def add_software(cytomine_host, cytomine_public_key, cytomine_private_key, id_project=None): #Connection to Cytomine Core conn = cytomine.Cytomine(cytomine_host, cytomine_public_key, cytomine_private_key, base_path = '/api/', working_path = '/tmp/', verbose= True) execute_command = "python algo/W_SpotDetection-Icy/wrapper.py --icy_scale3sensitivity $icy_scale3sensitivity " \ + "--cytomine_host $host " \ + "--cytomine_public_key $publicKey " \ + "--cytomine_private_key $privateKey " \ + "--cytomine_id_project $cytomine_id_project " #define software parameter template software = conn.add_software("W_SpotDetection-Icy", "createRabbitJobWithArgsService","ValidateAnnotation", execute_command) conn.add_software_parameter("icy_scale3sensitivity", software.id, "Number", 40, True, 10, False) #for logging (set by server) conn.add_software_parameter("cytomine_id_software", software.id, "Number",0, True, 400, True) conn.add_software_parameter("cytomine_id_project", software.id, "Number", 0, True, 500, True) if id_project: #add software to a given project addSoftwareProject = conn.add_software_project(id_project,software.id)
from cytomine.models import SoftwareParameter, SoftwareParameterCollection if __name__ == "__main__": import cytomine # Connect to cytomine, edit connection values cytomine_host = "demo.cytomine.be" cytomine_public_key = "274db678-5c0c-4497-beb6-6602a532aab8" # to complete cytomine_private_key = "4e2e1047-c706-47ad-8ba1-95aae6dabbf0" # to complete id_project = -1 # to complete # Connection to Cytomine Core conn = cytomine.Cytomine(cytomine_host, cytomine_public_key, cytomine_private_key, working_path=os.path.join(tempfile.gettempdir(), "cytomine"), verbose=True) # define software parameter template software = conn.add_software("Demo_SLDC_Workflow_With_Pyxit", "pyxitSuggestedTermJobService", "ValidateAnnotation") conn.add_software_parameter("cytomine_id_software", software.id, "Number", 0, True, 1, True) conn.add_software_parameter("cytomine_id_project", software.id, "Number", 0, True, 100, True) conn.add_software_parameter("cytomine_id_image", software.id, "Number", 0, True, 200, True) conn.add_software_parameter("n_jobs", software.id, "Number", 1, True, 300, False)
def main(): p = optparse.OptionParser( description='Cytomine Landmark Detection : Model building', prog='Cytomine Landmark Detector : Model builder', version='0.1') p.add_option( '--cytomine_host', type="string", default='beta.cytomine.be', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='XXX', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='YYY', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option( '--cytomine_training_images', type='string', default='all', dest='cytomine_training_images', help= "IDs of the training images. Must be separated by commas, no spaces. 'all' takes all the available annotated images." ) p.add_option( '--cytomine_id_terms', type='string', default=1, dest='cytomine_id_terms', help= "The identifiers of the terms to create detection models for. Terms must be separated by commas (no spaces). If 'all' is mentioned instead, every terms will be detected." ) p.add_option( '--image_type', type='string', default='jpg', dest='image_type', help="The type of the images that will be used (jpg, bmp, png,...)") p.add_option('--model_njobs', type='int', default=4, dest='model_njobs', help="The number of processors used for model building") p.add_option('--model_D_MAX', type='int', default=6, dest='model_D_MAX', help="D_MAX parameter.") p.add_option('--model_n_samples', type='int', default=200, dest='model_n_samples', help="Number of samples for phase 1.") p.add_option('--model_W', type='int', default=100, dest='model_W', help="Window size for feature extraction.") p.add_option('--model_n', type='int', default=20, dest='model_n', help="Number of samples extracted.") p.add_option('--model_T', type='int', default=50, dest='model_T', help="Number of trees for phase 1.") p.add_option('--model_step', type='int', default=3, dest='model_step', help="Step for prediction for phase 1.") p.add_option('--model_n_reduc', type='int', default=2, dest='model_n_reduc', help="Size for PCA reduction in phase 2.") p.add_option('--model_R_MAX', type='int', default=20, dest='model_R_MAX', help="Maximal radius for phase 2.") p.add_option('--model_R_MIN', type='int', default=3, dest='model_R_MIN', help="Minimal radius for phase 2.") p.add_option('--model_alpha', type='float', default=0.5, dest='model_alpha', help="Radius reduction parameter for phase 2.") p.add_option('--model_save_to', type='string', default='/tmp/', dest='model_save_to', help="Destination for model storage") p.add_option('--model_name', type='string', dest='model_name', help="Name of the model (used for saving)") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=sys.argv) options.cytomine_working_path = options.cytomine_working_path.rstrip( '/') + '/' cytomine_connection = cytomine.Cytomine( options.cytomine_host, options.cytomine_public_key, options.cytomine_private_key, base_path=options.cytomine_base_path, working_path=options.cytomine_working_path, verbose=options.verbose) current_user = cytomine_connection.get_current_user() if not current_user.algo: user_job = cytomine_connection.add_user_job( options.cytomine_id_software, options.cytomine_id_project) cytomine_connection.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) else: user_job = current_user job = cytomine_connection.get_job(user_job.job) cytomine_connection.update_job_status(job, status=job.RUNNING, progress=0, status_comment="Bulding model...") model_repo = options.model_save_to if not os.path.isdir(model_repo): os.mkdir(model_repo) download_images(cytomine_connection, options.cytomine_id_project) download_annotations(cytomine_connection, options.cytomine_id_project, options.cytomine_working_path) repository = options.cytomine_working_path + str( options.cytomine_id_project) + '/' (xc, yc, xr, yr, ims, term_to_i, i_to_term) = getallcoords(repository.rstrip('/') + '/txt/') (nims, nldms) = xc.shape if options.cytomine_id_terms != 'all': term_list = [ int(term) for term in options.cytomine_id_terms.split(',') ] Xc = np.zeros((nims, len(term_list))) Yc = np.zeros(Xc.shape) i = 0 for id_term in term_list: Xc[:, i] = xc[:, term_to_i[id_term]] Yc[:, i] = yc[:, term_to_i[id_term]] i += 1 else: term_list = term_to_i.keys() Xc = xc Yc = yc (nims, nldms) = Xc.shape if options.cytomine_training_images != 'all': im_list = [int(p) for p in options.cytomine_training_images.split(',')] else: im_list = ims X = np.zeros((len(im_list), nldms)) Y = np.zeros((len(im_list), nldms)) im_to_i = {} for i in range(nims): im_to_i[ims[i]] = i for i in range(len(im_list)): X[i, :] = Xc[im_to_i[im_list[i]], :] Y[i, :] = Yc[im_to_i[im_list[i]], :] Xc = X Yc = Y h2 = generate_2_horizontal(options.model_W, options.model_n) v2 = generate_2_vertical(options.model_W, options.model_n) h3 = generate_3_horizontal(options.model_W, options.model_n) v3 = generate_3_vertical(options.model_W, options.model_n) sq = generate_square(options.model_W, options.model_n) joblib.dump((h2, v2, h3, v3, sq), '%s%s_lc_feature_map.pkl' % (model_repo, options.model_name)) for id_term in term_list: (dataset, rep, img) = build_dataset_image_offset_mp( repository, Xc[:, term_to_i[id_term]], Yc[:, term_to_i[id_term]], im_list, options.model_D_MAX, options.model_n_samples, h2, v2, h3, v3, sq, options.model_njobs) clf = VotingTreeRegressor(n_estimators=options.model_T, n_jobs=options.model_njobs) clf = clf.fit(dataset, rep) joblib.dump( clf, '%s%s_lc_regressor_%d.pkl' % (model_repo, options.model_name, id_term)) xt = procrustes(Xc, Yc) (mu, P) = apply_pca(xt, options.model_n_reduc) joblib.dump((mu, P, np.mean(Xc, axis=0), np.mean(Yc, axis=0)), '%s%s_lc_pca.pkl' % (model_repo, options.model_name)) F = open( '%s%s_lc_parameters.conf' % (options.model_save_to, options.model_name), 'wb') F.write('cytomine_id_terms %s\n' % options.cytomine_id_terms) F.write('model_njobs %d\n' % options.model_njobs) F.write('model_D_MAX %d\n' % options.model_D_MAX) F.write('model_n_samples %d\n' % options.model_n_samples) F.write('model_W %d\n' % options.model_W) F.write('model_n %d\n' % options.model_n) F.write('model_T %d\n' % options.model_T) F.write('model_step %d\n' % options.model_step) F.write('model_n_reduc %d\n' % options.model_n_reduc) F.write('model_R_MAX %d\n' % options.model_R_MAX) F.write('model_R_MIN %d\n' % options.model_R_MIN) F.write('model_alpha %f\n' % options.model_alpha) F.close()
def main(): p = optparse.OptionParser( description='Cytomine Landmark Detection : Model building', prog='Cytomine Landmark Detector : Model builder', version='0.1') p.add_option( '--cytomine_host', type="string", default='beta.cytomine.be', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='XXX', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='YYY', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option( '--cytomine_training_images', default="all", type="string", dest="cytomine_training_images", help= "identifiers of the images used to create the models. ids must be separated by commas (no spaces). If 'all' is mentioned instead, every image with manual annotation will be used." ) p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option( '--image_type', type='string', default='jpg', dest='image_type', help="The type of the images that will be used (jpg, bmp, png,...)") p.add_option('--model_njobs', type='int', default=1, dest='model_njobs', help="The number of processors used for model building") p.add_option( '--cytomine_id_terms', type='string', default=1, dest='cytomine_id_terms', help= "The identifiers of the terms to create detection models for. Terms must be separated by commas (no spaces). If 'all' is mentioned instead, every terms will be detected." ) p.add_option('--model_NT_P1', type='int', default=6, dest='model_NT_P1', help="Number of trees for phase 1.") p.add_option('--model_F_P1', type='int', default=200, dest='model_F_P1', help="Number of features for phase 1.") p.add_option('--model_R_P1', type='int', default=3, dest='model_R_P1', help="Radius in which phase 1 samples are extracted.") p.add_option('--model_sigma', type='int', default=20, dest='model_sigma', help="Standard deviation for the gaussian.") p.add_option('--model_delta', type='float', default=3, dest='model_delta', help="Resizing factor.") p.add_option('--model_P', type='float', default=3, dest='model_P', help="Proportion of non-landmarks.") p.add_option('--model_R_P2', type='int', default=3, dest='model_R_P2', help="Radius in which phase 2 samples are extracted.") p.add_option('--model_ns_P2', type='int', default=3, dest='model_ns_P2', help="Number of samples for phase 2.") p.add_option('--model_NT_P2', type='int', default=3, dest='model_NT_P2', help="Number of trees for phase 2.") p.add_option('--model_F_P2', type='int', default=3, dest='model_F_P2', help="Number of features for phase 2.") p.add_option('--model_filter_size', type='int', default=3, dest='model_filter_size', help="Size of the filter for phase 2.") p.add_option('--model_beta', type='float', default=3, dest='model_beta', help="Beta for phase 2.") p.add_option('--model_n_iterations', type='int', default=3, dest='model_n_iterations', help="Number of iterations for phase 2.") p.add_option('--model_ncandidates', type='int', default=3, dest='model_ncandidates', help="Number of candidates for phase 3.") p.add_option('--model_sde', type='float', default=10., dest='model_sde', help="Standard deviation for gaussian phase 3.") p.add_option('--model_T', type='int', default=3, dest='model_T', help="Number of edges for phase 3.") p.add_option('--model_save_to', type='string', default='/tmp/', dest='model_save_to', help="Destination for model storage") p.add_option('--model_name', type='string', dest='model_name', help="Name of the model (used for saving)") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") opt_parser, arguments = p.parse_args(args=sys.argv) opt_parser.cytomine_working_path = opt_parser.cytomine_working_path.rstrip( '/') + '/' cytomine_connection = cytomine.Cytomine( opt_parser.cytomine_host, opt_parser.cytomine_public_key, opt_parser.cytomine_private_key, base_path=opt_parser.cytomine_base_path, working_path=opt_parser.cytomine_working_path, verbose=str2bool(opt_parser.verbose)) current_user = cytomine_connection.get_current_user() if not current_user.algo: user_job = cytomine_connection.add_user_job( opt_parser.cytomine_id_software, opt_parser.cytomine_id_project) cytomine_connection.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) else: user_job = current_user run_by_user_job = True job = cytomine_connection.get_job(user_job.job) cytomine_connection.update_job_status(job, status=job.RUNNING, progress=0, status_comment="Bulding model...") job_parameters = {} job_parameters['cytomine_id_terms'] = opt_parser.cytomine_id_terms job_parameters['model_njobs'] = opt_parser.model_njobs job_parameters['model_NT_P1'] = opt_parser.model_NT_P1 job_parameters['model_F_P1'] = opt_parser.model_F_P1 job_parameters['model_R_P1'] = opt_parser.model_R_P1 job_parameters['model_sigma'] = opt_parser.model_sigma job_parameters['model_delta'] = opt_parser.model_delta job_parameters['model_P'] = opt_parser.model_P job_parameters['model_R_P2'] = opt_parser.model_R_P2 job_parameters['model_ns_P2'] = opt_parser.model_ns_P2 job_parameters['model_NT_P2'] = opt_parser.model_NT_P2 job_parameters['model_F_P2'] = opt_parser.model_F_P2 job_parameters['model_filter_size'] = opt_parser.model_filter_size job_parameters['model_beta'] = opt_parser.model_beta job_parameters['model_n_iterations'] = opt_parser.model_n_iterations job_parameters['model_ncandidates'] = opt_parser.model_ncandidates job_parameters['model_sde'] = opt_parser.model_sde job_parameters['model_T'] = opt_parser.model_T model_repo = opt_parser.model_save_to if not os.path.isdir(model_repo): os.mkdir(model_repo) if not run_by_user_job: cytomine_connection.add_job_parameters( user_job.job, cytomine_connection.get_software(opt_parser.cytomine_id_software), job_parameters) download_images(cytomine_connection, opt_parser.cytomine_id_project) download_annotations(cytomine_connection, opt_parser.cytomine_id_project, opt_parser.cytomine_working_path) repository = opt_parser.cytomine_working_path + str( opt_parser.cytomine_id_project) + '/' (xc, yc, xr, yr, ims, term_to_i, i_to_term) = getallcoords(repository.rstrip('/') + '/txt/') (nims, nldms) = xc.shape if opt_parser.cytomine_id_terms != 'all': term_list = np.sort( [int(term) for term in opt_parser.cytomine_id_terms.split(',')]) Xc = np.zeros((nims, len(term_list))) Yc = np.zeros(Xc.shape) i = 0 for id_term in term_list: Xc[:, i] = xc[:, term_to_i[id_term]] Yc[:, i] = yc[:, term_to_i[id_term]] i += 1 else: term_list = np.sort(term_to_i.keys()) Xc = xc Yc = yc if opt_parser.cytomine_training_images == 'all': tr_im = ims else: tr_im = [ int(p) for p in opt_parser.cytomine_training_images.split(',') ] (dataset, rep, img, feature_offsets_1) = build_phase_1_model(repository, tr_image=tr_im, image_ids=ims, n_jobs=opt_parser.model_njobs, NT=opt_parser.model_NT_P1, F=opt_parser.model_F_P1, R=opt_parser.model_R_P1, sigma=opt_parser.model_sigma, delta=opt_parser.model_delta, P=opt_parser.model_P, X=Xc, Y=Yc) clf = SeparateTrees(n_estimators=opt_parser.model_NT_P1, n_jobs=opt_parser.model_njobs) clf = clf.fit(dataset, rep) joblib.dump( clf, '%s%s_classifier_phase1.pkl' % (model_repo, opt_parser.model_name)) joblib.dump( feature_offsets_1, '%s%s_offsets_phase1.pkl' % (model_repo, opt_parser.model_name)) for id_term in term_list: (dataset, rep, number, feature_offsets_2) = build_phase_2_model( repository, tr_image=tr_im, image_ids=ims, n_jobs=opt_parser.model_njobs, IP=id_term, NT=opt_parser.model_NT_P2, F=opt_parser.model_F_P2, R=opt_parser.model_R_P2, N=opt_parser.model_ns_P2, sigma=opt_parser.model_sigma, delta=opt_parser.model_delta) reg = SeparateTreesRegressor(n_estimators=opt_parser.model_NT_P2, n_jobs=opt_parser.model_njobs) reg.fit(dataset, rep) joblib.dump( reg, '%s%s_dmbl_regressor_phase2_%d.pkl' % (model_repo, opt_parser.model_name, id_term)) joblib.dump( feature_offsets_2, '%s%s_dmbl_offsets_phase2_%d.pkl' % (model_repo, opt_parser.model_name, id_term)) (nims, nldms) = xc.shape X = np.zeros((len(tr_im), nldms)) Y = np.zeros(X.shape) j = 0 for i in range(nims): if ims[i] in tr_im: X[j, :] = xc[i, :] Y[j, :] = yc[i, :] j += 1 edges = build_edgematrix_phase_3(X, Y, opt_parser.model_sde, opt_parser.model_delta, opt_parser.model_T) joblib.dump( edges, '%s%s_edgematrix_phase3.pkl' % (opt_parser.model_save_to, opt_parser.model_name)) F = open( '%s%s_dmbl_parameters.conf' % (opt_parser.model_save_to, opt_parser.model_name), 'wb') F.write('cytomine_id_terms %s\n' % opt_parser.cytomine_id_terms) F.write('model_njobs %d\n' % opt_parser.model_njobs) F.write('model_NT_P1 %d\n' % opt_parser.model_NT_P1) F.write('model_F_P1 %d\n' % opt_parser.model_F_P1) F.write('model_R_P1 %d\n' % opt_parser.model_R_P1) F.write('model_sigma %f\n' % opt_parser.model_sigma) F.write('model_delta %f\n' % opt_parser.model_delta) F.write('model_P %f\n' % opt_parser.model_P) F.write('model_R_P2 %d\n' % opt_parser.model_R_P2) F.write('model_ns_P2 %d\n' % opt_parser.model_ns_P2) F.write('model_NT_P2 %d\n' % opt_parser.model_NT_P2) F.write('model_F_P2 %d\n' % opt_parser.model_F_P2) F.write('model_filter_size %d\n' % opt_parser.model_filter_size) F.write('model_beta %f\n' % opt_parser.model_beta) F.write('model_n_iterations %d\n' % opt_parser.model_n_iterations) F.write("model_ncandidates %d\n" % opt_parser.model_ncandidates) F.write('model_sde %f\n' % opt_parser.model_sde) F.write('model_T %d' % opt_parser.model_T) F.close()
def main(argv): # Define command line options p = optparse.OptionParser( description='Pyxit/Cytomine Classification Model Prediction', prog='PyXit Classification Model Prediction (PYthon piXiT)') p.add_option( "--cytomine_host", type="string", default='', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--pyxit_save_to', type='string', dest='pyxit_save_to', help="pyxit model file") # future: get it from server db p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-i', '--cytomine_id_image', type="int", dest="cytomine_id_image", help="The Cytomine image identifier") p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('--cytomine_dump_type', type='int', dest='cytomine_dump_type', help="dump type of annotations (with/out alpha channel)") p.add_option( '--cytomine_id_userjob', type="int", dest="cytomine_id_userjob", help= "The Cytomine user (job) id of annotations to classify with the model") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=argv) parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_software'] = options.cytomine_id_software # to define which annotations we will classify parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_image'] = options.cytomine_id_image parameters['cytomine_id_userjob'] = options.cytomine_id_userjob # to define which resolution and image type we will classify parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['cytomine_dump_type'] = options.cytomine_dump_type parameters['pyxit_save_to'] = options.pyxit_save_to print "[pyxit.main] Options = ", options # Create JOB/USER/JOB conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) # Create a new userjob if connected as human user current_user = conn.get_current_user() run_by_user_job = False if current_user.algo == False: print "adduserJob..." user_job = conn.add_user_job(parameters['cytomine_id_software'], parameters['cytomine_id_project']) print "set_credentials..." conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) print "done" else: user_job = current_user print "Already running as userjob" run_by_user_job = True job = conn.get_job(user_job.job) print "Fetching data..." job = conn.update_job_status( job, status_comment="Publish software parameters values") if run_by_user_job == False: job_parameters_values = conn.add_job_parameters( user_job.job, conn.get_software(parameters['cytomine_id_software']), parameters) job = conn.update_job_status(job, status=job.RUNNING, status_comment="Run...", progress=0) # Image dump type (for classification use 1) if parameters['cytomine_dump_type'] == 1: annotation_get_func = Annotation.get_annotation_crop_url elif parameters['cytomine_dump_type'] == 2: annotation_get_func = Annotation.get_annotation_alpha_crop_url else: print "default annotation type crop" annotation_get_func = Annotation.get_annotation_crop_url # Get description of annotations to predict (e.g. geometries created by a another object finder (e.g. threshold) job) job = conn.update_job_status(job, status=job.RUNNING, status_comment="Run (getting annotations)...", progress=25) candidate_annotations = conn.get_annotations( id_user=parameters['cytomine_id_userjob'], id_image=parameters['cytomine_id_image'], id_project=parameters['cytomine_id_project'], showWKT=True, showMeta=True) print "Number of annotations to predict: %d" % len( candidate_annotations.data()) time.sleep(2) # Create temporary dir to download annotation crops folder_name = "%s/annotations/project-%d/crops-candidates-%d-%d/zoom-%d/" % ( parameters["cytomine_working_path"], parameters["cytomine_id_project"], parameters["cytomine_id_image"], job.userJob, # current job parameters['cytomine_zoom_level']) if not os.path.exists(folder_name): os.makedirs(folder_name) # Load Classifier model classifier = open(parameters['pyxit_save_to'], "r") classes = pickle.load(classifier) pyxit = pickle.load(classifier) print "Model: %s" % pyxit time.sleep(2) print "Dumping annotation cropped images to classify to %s" % folder_name annotation_mapping = {} for i, annotation in enumerate(candidate_annotations.data()): url = annotation_get_func( annotation, desired_zoom=parameters['cytomine_zoom_level']) filename = folder_name + str(annotation.id) + ".png" annotation_mapping[annotation.id] = filename conn.fetch_url_into_file(url, filename, False, True) np_image = cv2.imread(filename, -1) if parameters['cytomine_dump_type'] == 2 and np_image is not None: alpha = np.array(np_image[:, :, 3]) image = np.array(np_image[:, :, 0:3]) image[alpha == 0] = (255, 255, 255) # to replace alpha by white cv2.imwrite(filename, image) print "Building subwindows from ", os.path.dirname( os.path.dirname(folder_name)) # Extract subwindows from all candidates annotations X, y = pyxitstandalone.build_from_dir( os.path.dirname(os.path.dirname(folder_name))) # Apply pyxit classifier model to X (parameters are already reused from model pkl file) job = conn.update_job_status(job, status=job.RUNNING, status_comment="Run (applying Classifier)...", progress=50) y_proba = pyxit.predict_proba(X) y_predict = classes.take(np.argmax(y_proba, axis=1), axis=0) y_rate = np.max(y_proba, axis=1) # Creating new annotations on Cytomine with predicted terms by current classifier model for k, annotation in enumerate(candidate_annotations.data()): filename = annotation_mapping[annotation.id] j = np.where(X == filename)[0][0] print "Annotation filename %s id: %d class: %d proba: %d" % ( filename, annotation.id, int(y_predict[j]), y_rate[j]) new_annotation = conn.add_annotation(annotation.location, parameters["cytomine_id_image"]) conn.add_annotation_term( new_annotation.id, int(y_predict[j]), int(y_predict[j]), y_rate[j], annotation_term_model=models.AlgoAnnotationTerm) job = conn.update_job_status(job, status=job.TERMINATED, progress=100, status_comment="Finish Job..") print "END."
if __name__ == "__main__": import cytomine # Connect to cytomine, edit connection values cytomine_host = "http://localhost-core:8080" Pk = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' Prk = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' id_project = 19941904 # Connection to Cytomine Core conn = cytomine.Cytomine(cytomine_host, Pk, Prk, base_path='/api/', working_path=os.path.join(tempfile.gettempdir(), "cytomine"), verbose=True) #add software with execute command software = conn.add_software( "Simple_Elastix", "createRabbitJobWithArgsService", "download", "/root/miniconda2/envs/cytomine/bin/python software_routeur/algo/simple_elastix/get_and_move.py " + "--cytomine_host $cytomine_host " + "--cytomine_public_key $cytomine_public_key " + "--cytomine_private_key $cytomine_private_key " + "--cytomine_id_software $cytomine_id_software " + "--cytomine_working_path software_router/algo/simple_elastix " + "--cytomine_id_project $cytomine_id_project " + "--fix_image_id fix_image " + "--mov_image_id $id_mov_image " +
def main(argv): parser = optparse.OptionParser(description='Cytomine Datamining', prog='cytomining', version='cytomining 0.1') parser.add_option('--cytomine_host', dest='cytomine_host', help='cytomine_host') parser.add_option('--cytomine_public_key', dest='cytomine_public_key', help='cytomine_public_key') parser.add_option('--cytomine_private_key', dest='cytomine_private_key', help='cytomine_private_key') parser.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") parser.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") parser.add_option('--cytomine_base_path', dest='cytomine_base_path', help='cytomine base path') parser.add_option('--cytomine_working_path', dest='cytomine_working_path', help='cytomine_working_path base path') options, arguments = parser.parse_args(args=argv) # copy options parameters = {} parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path print "Connection to Cytomine server" cytomine_connection = cytomine.Cytomine( parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path']) # Create a new userjob if connected as human user current_user = cytomine_connection.get_current_user() if current_user.algo == False: print "adduserJob..." user_job = cytomine_connection.add_user_job( parameters['cytomine_id_software'], parameters['cytomine_id_project']) print "set_credentials..." cytomine_connection.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) print "done" else: user_job = current_user print "Already running as userjob" job = cytomine_connection.get_job(user_job.job) job = cytomine_connection.update_job_status(job, status=job.RUNNING, status_comment="Fetching data", progress=0) images = cytomine_connection.get_project_image_instances( parameters['cytomine_id_project']) images = images.data() xpos = {} ypos = {} terms = {} progress = 0 delta = 80 / len(images) for image in images: progress += delta job = cytomine_connection.update_job_status( job, status=job.RUNNING, status_comment="Fetching data", progress=progress) annotations = cytomine_connection.get_annotations( id_project=parameters['cytomine_id_project'], showWKT=True, id_image=image.id, reviewed_only=True) ann_data = annotations.data() for ann in ann_data: if (len(ann.term) > 0): l = ann.location if (l.rfind('POINT') == -1): pol = shapely.wkt.loads(l) poi = pol.centroid else: poi = shapely.wkt.loads(l) (cx, cy) = poi.xy xpos[(ann.term[0], image.id)] = int(cx[0]) ypos[(ann.term[0], image.id)] = image.height - int(cy[0]) term = cytomine_connection.get_term(ann.term[0]) terms[term.id] = term.name key_t = terms.keys() job = cytomine_connection.update_job_status(job, status=job.RUNNING, status_comment="Write in file", progress=90) if not os.path.exists(parameters['cytomine_working_path']): os.makedirs(parameters['cytomine_working_path']) csv = open( '%s%s.csv' % (parameters['cytomine_working_path'], str(parameters['cytomine_id_project'])), 'w') csv.write('ID_IMAGE;') for i in range(len(key_t)): csv.write('%s_x;%s_y;' % (str(terms[key_t[i]]), str(terms[key_t[i]]))) csv.write('\n') for image in images: csv.write('%s;' % str(image.id)) for i in range(len(key_t)): if ((key_t[i], image.id) in xpos): csv.write( '%3.3f;%3.3f;' % (xpos[(key_t[i], image.id)], ypos[(key_t[i], image.id)])) else: csv.write('-1;-1;') csv.write('\n') csv.close() job_data = cytomine_connection.add_job_data( job, key=job.id, filename='%s%s.csv' % (parameters['cytomine_working_path'], str(parameters['cytomine_id_project']))) job = cytomine_connection.update_job_status( job, status=job.TERMINATED, status_comment="File available", progress=100) print "File available"
def main(argv): # Define command line options p = optparse.OptionParser( description='Pyxit/Cytomine Classification Model Builder', prog='PyXit Classification Model Builder (PYthon piXiT)') p.add_option( "--cytomine_host", type="string", default='', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('--cytomine_dump_type', type='int', dest='cytomine_dump_type', help="annotation type (1=crop, 2=alphamask)") p.add_option('--cytomine_annotation_projects', type="string", dest="cytomine_annotation_projects", help="Projects from which annotations are extracted") p.add_option( '--cytomine_predict_terms', type='string', default='0', dest='cytomine_predict_terms', help="term ids of predicted terms (=positive class in binary mode)") p.add_option('--cytomine_excluded_terms', type='string', default='0', dest='cytomine_excluded_terms', help="term ids of excluded terms") #p.add_option('--cytomine_reviewed', default=False, action="store_true", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--cytomine_reviewed', type='string', default="False", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--pyxit_target_width', type='int', dest='pyxit_target_width', help="pyxit subwindows width") p.add_option('--pyxit_target_height', type='int', dest='pyxit_target_height', help="pyxit subwindows height") p.add_option('--pyxit_save_to', type='string', dest='pyxit_save_to', help="pyxit model directory") #future: get it from server db p.add_option( '--pyxit_colorspace', type='int', dest='pyxit_colorspace', help="pyxit colorspace encoding") #future: get it from server db p.add_option( '--pyxit_n_jobs', type='int', dest='pyxit_n_jobs', help="pyxit number of jobs for trees") #future: get it from server db p.add_option('--pyxit_n_subwindows', default=10, type="int", dest="pyxit_n_subwindows", help="number of subwindows") p.add_option('--pyxit_min_size', default=0.5, type="float", dest="pyxit_min_size", help="min size") p.add_option('--pyxit_max_size', default=1.0, type="float", dest="pyxit_max_size", help="max size") p.add_option('--pyxit_interpolation', default=2, type="int", dest="pyxit_interpolation", help="interpolation method 1,2,3,4") #p.add_option('--pyxit_transpose', default=False, action="store_true", dest="pyxit_transpose", help="transpose subwindows") p.add_option('--pyxit_transpose', type="string", default="False", dest="pyxit_transpose", help="transpose subwindows") #p.add_option('--pyxit_fixed_size', default=False, action="store_true", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--pyxit_fixed_size', type="string", default="False", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--forest_n_estimators', default=10, type="int", dest="forest_n_estimators", help="number of base estimators (T)") p.add_option('--forest_max_features', default=1, type="int", dest="forest_max_features", help="max features at test node (k)") p.add_option('--forest_min_samples_split', default=1, type="int", dest="forest_min_samples_split", help="minimum node sample size (nmin)") p.add_option('--forest_shared_mem', default=False, action="store_true", dest="forest_shared_mem", help="shared mem") p.add_option( '--svm', default=0, dest="svm", help= "final svm classifier: 0=nosvm, 1=libsvm, 2=liblinear, 3=lr-l1, 4=lr-l2", type="int") p.add_option('--svm_c', default=1.0, type="float", dest="svm_c", help="svm C") #p.add_option('--verbose', action="store_true", default=True, dest="verbose", help="Turn on verbose mode") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=argv) parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_annotation_projects'] = map( int, options.cytomine_annotation_projects.split(',')) parameters['cytomine_predict_terms'] = map( int, options.cytomine_predict_terms.split(',')) parameters['cytomine_excluded_terms'] = map( int, options.cytomine_excluded_terms.split(',')) parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['cytomine_dump_type'] = options.cytomine_dump_type parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed) pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators pyxit_parameters['forest_max_features'] = options.forest_max_features pyxit_parameters[ 'forest_min_samples_split'] = options.forest_min_samples_split pyxit_parameters['forest_shared_mem'] = options.forest_min_samples_split pyxit_parameters['svm'] = options.svm pyxit_parameters['svm_c'] = options.svm_c pyxit_parameters['pyxit_save_to'] = options.pyxit_save_to pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs # Check for errors in the options if options.verbose: print "[pyxit.main] Options = ", options # Create JOB/USER/JOB conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) #Create a new userjob if connected as human user current_user = conn.get_current_user() run_by_user_job = False if current_user.algo == False: print "adduserJob..." user_job = conn.add_user_job(parameters['cytomine_id_software'], parameters['cytomine_id_project']) print "set_credentials..." conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) print "done" else: user_job = current_user print "Already running as userjob" run_by_user_job = True job = conn.get_job(user_job.job) print "Fetching data..." job = conn.update_job_status(job, status=job.RUNNING, status_comment="Run...", progress=0) #Get annotation descriptions (JSON) from project(s) annotations = None for prj in parameters['cytomine_annotation_projects']: if parameters["cytomine_reviewed"]: print "Retrieving reviewed annotations..." annotations_prj = conn.get_annotations( id_project=prj, reviewed_only=parameters["cytomine_reviewed"]) print "Reviewed annotations: %d" % len(annotations_prj.data()) else: print "Retrieving (unreviewed) annotations..." annotations_prj = conn.get_annotations(id_project=prj) print "(Unreviewed) annotations: %d" % len(annotations_prj.data()) if not annotations: annotations = annotations_prj else: annotations.data().extend(annotations_prj.data()) print "Nb annotations so far... = %d" % len(annotations.data()) print "Total annotations projects %s = %d" % ( parameters['cytomine_annotation_projects'], len(annotations.data())) time.sleep(2) #Set output dir parameters pyxit_parameters['dir_ls'] = os.path.join( parameters["cytomine_working_path"], str(parameters['cytomine_annotation_projects']).replace( ',', '-').replace('[', '').replace(']', '').replace(' ', ''), "zoom_level", str(parameters['cytomine_zoom_level']), "dump_type", str(parameters['cytomine_dump_type'])) if not os.path.exists(pyxit_parameters['dir_ls']): print "Creating annotation directory: %s" % pyxit_parameters['dir_ls'] os.makedirs(pyxit_parameters['dir_ls']) job = conn.update_job_status(job, status=job.RUNNING, status_comment="Dump annotations...", progress=50) time.sleep(2) #Image dump type (for classification use 1) if (parameters['cytomine_dump_type'] == 1): annotation_get_func = Annotation.get_annotation_crop_url elif (parameters['cytomine_dump_type'] == 2): annotation_get_func = Annotation.get_annotation_alpha_crop_url else: print "default annotation type crop" annotation_get_func = Annotation.get_annotation_crop_url #Dump annotation images locally annotations = conn.dump_annotations( annotations=annotations, get_image_url_func=annotation_get_func, dest_path=pyxit_parameters['dir_ls'], desired_zoom=parameters['cytomine_zoom_level'], excluded_terms=parameters['cytomine_excluded_terms']) print "END dump annotations" job = conn.update_job_status(job, status=job.RUNNING, status_comment="Build model...", progress=75) #Build pyxit classification model (and saves it) print "Upload software parameters values to Cytomine-Core..." if run_by_user_job == False: parameters_values = conn.add_job_parameters( user_job.job, conn.get_software(parameters['cytomine_id_software']), pyxit_parameters) argv = parameters_values_to_argv(pyxit_parameters, parameters_values) else: argv = [] for key in pyxit_parameters: value = pyxit_parameters[key] if type(value) is bool or value == 'True': if bool(value): argv.append("--%s" % key) elif not value == 'False': argv.append("--%s" % key) argv.append("%s" % value) print "argv :" print argv print "Run PyXiT..." d = os.path.dirname(pyxit_parameters['pyxit_save_to']) if not os.path.exists(d): os.makedirs(d) predict = pyxitstandalone.main(argv) print "-------------------------------------------------------" print "Pyxit Classification Model saved locally: %s " % pyxit_parameters[ 'pyxit_save_to'] print "-------------------------------------------------------" job = conn.update_job_status(job, status=job.TERMINATED, status_comment="Finish", progress=100)
def main(argv): current_path = os.getcwd() + '/' + os.path.dirname(__file__) # Define command line options p = optparse.OptionParser(description='Cytomine Segmentation prediction', prog='Cytomine segmentation prediction', version='0.1') p.add_option( '--cytomine_host', type="string", default='beta.cytomine.be', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-i', '--cytomine_id_image', type='int', dest='cytomine_id_image', help="image id from cytomine", metavar='IMAGE') p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('-j', '--nb_jobs', type='int', dest='nb_jobs', help="number of parallel jobs") p.add_option( '--cytomine_predict_terms', type='str', dest='cytomine_predict_terms', help= "term id of all positive terms. The first term is the output predicted annotation term" ) p.add_option('--cytomine_excluded_terms', type='string', dest='cytomine_excluded_terms', help="term id of excluded terms)") p.add_option('--pyxit_target_width', type='int', dest='pyxit_target_width', help="pyxit subwindows width") p.add_option('--pyxit_target_height', type='int', dest='pyxit_target_height', help="pyxit subwindows height") p.add_option('--pyxit_colorspace', type='int', dest='pyxit_colorspace', help="pyxit colorspace encoding") p.add_option('--pyxit_nb_jobs', type='int', dest='pyxit_nb_jobs', help="pyxit number of jobs for trees") p.add_option('--pyxit_fixed_size', type='string', default="0", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--pyxit_transpose', type='string', default="0", dest="pyxit_transpose", help="transpose subwindows") p.add_option('--pyxit_n_subwindows', type='int', default="10", dest="pyxit_n_subwindows", help="number of subwindows") p.add_option('--pyxit_interpolation', default=2, type="int", dest="pyxit_interpolation", help="interpolation method 1,2,3,4") p.add_option('--pyxit_min_size', default=0.5, type="float", dest="pyxit_min_size", help="min size") p.add_option('--pyxit_max_size', default=1.0, type="float", dest="pyxit_max_size", help="max size") p.add_option('--cytomine_reviewed', type='string', default="False", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--cytomine_dump_annotations', type='string', default="0", dest="cytomine_dump_annotations", help="Dump training annotations or not") p.add_option('--cytomine_dump_annotation_stats', type='string', default="0", dest="cytomine_dump_annotation_stats", help="Calculate stats on dumped annotations or not") p.add_option('--build_model', type="string", default="0", dest="build_model", help="Turn on (1) or off (0) model building") p.add_option('--cytomine_annotation_projects', type="string", dest="cytomine_annotation_projects", help="Projects from which annotations are extracted") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") p.add_option('--keras_save_to', type='string', default="", dest='keras_save_to', help="keras model weight file") p.add_option('--keras_batch_size', type="int", dest="keras_batch_size", help="Training batch size") p.add_option('--keras_n_epochs', type="int", dest="keras_n_epochs", help="Number of epochs") p.add_option('--keras_shuffle', type="string", dest="keras_shuffle", help="Turn on (1) or off (0) batch shuffle") p.add_option('--keras_validation_split', type="float", dest="keras_validation_split", help="Batch validation split") options, arguments = p.parse_args(args=argv) parameters = {} parameters['keras_save_to'] = options.keras_save_to parameters['keras_batch_size'] = options.keras_batch_size parameters['keras_n_epochs'] = options.keras_n_epochs parameters['keras_shuffle'] = options.keras_shuffle parameters['keras_validation_split'] = options.keras_validation_split parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_predict_terms'] = map( int, options.cytomine_predict_terms.split(',')) parameters['cytomine_predicted_annotation_term'] = parameters[ 'cytomine_predict_terms'][0] parameters['cytomine_excluded_terms'] = map( int, options.cytomine_excluded_terms.split(',')) parameters['pyxit_colorspace'] = options.pyxit_colorspace parameters['pyxit_nb_jobs'] = options.pyxit_nb_jobs parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs parameters['cytomine_nb_jobs'] = options.pyxit_nb_jobs parameters['cytomine_id_image'] = options.cytomine_id_image parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['nb_jobs'] = options.nb_jobs parameters['pyxit_target_width'] = options.pyxit_target_width parameters['pyxit_target_height'] = options.pyxit_target_height parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows parameters['pyxit_interpolation'] = options.pyxit_interpolation parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) parameters['pyxit_min_size'] = options.pyxit_min_size parameters['pyxit_max_size'] = options.pyxit_max_size parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) parameters['cytomine_annotation_projects'] = map( int, options.cytomine_annotation_projects.split(',')) parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed) parameters['cytomine_dump_annotation_stats'] = str2bool( options.cytomine_dump_annotation_stats) parameters['cytomine_dump_annotations'] = str2bool( options.cytomine_dump_annotations) parameters['build_model'] = str2bool(options.build_model) parameters['dir_ls'] = os.path.join( parameters["cytomine_working_path"], str(parameters['cytomine_annotation_projects']).replace( ',', '-').replace('[', '').replace(']', '').replace(' ', ''), "zoom_level", str(parameters['cytomine_zoom_level'])) pyxit_parameters = {} pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) pyxit_parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs if options.verbose: print(parameters) # Create Cytomine connection conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) # Dump annotations if parameters['cytomine_dump_annotations']: # Get annotation descriptions (JSON) from project(s) annotations = None for prj in parameters['cytomine_annotation_projects']: if parameters["cytomine_reviewed"]: annotations_prj = conn.get_annotations( id_project=prj, reviewed_only=parameters["cytomine_reviewed"]) else: annotations_prj = conn.get_annotations(id_project=prj) if not annotations: annotations = annotations_prj else: annotations.data().extend(annotations_prj.data()) if prj == 21907448 or prj == 155194683: annotations_prj = conn.get_annotations(id_project=prj, id_term=91376951) annotations.data().extend(annotations_prj.data()) print("Nb annotations so far... = %d" % len(annotations.data())) print("Total annotations projects %s = %d" % (parameters['cytomine_annotation_projects'], len(annotations.data()))) # Set output dir parameters if not os.path.exists(parameters['dir_ls']): print("Creating annotation directory: %s" % parameters['dir_ls']) os.makedirs(parameters['dir_ls']) # Dump annotation images locally print("Dump training annotation images in %s...", parameters['dir_ls']) conn.dump_annotations( annotations=annotations, get_image_url_func=Annotation.get_annotation_alpha_crop_url, dest_path=parameters['dir_ls'], desired_zoom=parameters['cytomine_zoom_level'], excluded_terms=parameters['cytomine_excluded_terms']) # Put positive terms under the same term and same for negative terms term_directories = os.listdir(parameters['dir_ls']) pos_image_path = os.path.join(parameters['dir_ls'], "image", "1") pos_mask_path = os.path.join(parameters['dir_ls'], "mask", "1") neg_image_path = os.path.join(parameters['dir_ls'], "image", "0") neg_mask_path = os.path.join(parameters['dir_ls'], "mask", "0") if not os.path.exists(pos_image_path): print("Creating positive annotation directory: %s" % pos_image_path) os.makedirs(pos_image_path) if not os.path.exists(neg_image_path): print("Creating negative annotation directory: %s" % neg_image_path) os.makedirs(neg_image_path) if not os.path.exists(pos_mask_path): print("Creating positive annotation directory: %s" % pos_mask_path) os.makedirs(pos_mask_path) if not os.path.exists(neg_mask_path): print("Creating negative annotation directory: %s" % neg_mask_path) os.makedirs(neg_mask_path) for dir in term_directories: dir_abs = os.path.join(parameters['dir_ls'], dir) # Move files print("Term directory: %s" % dir_abs) if int(dir) in parameters['cytomine_predict_terms']: print("Positive term") for image_file in os.listdir(dir_abs): print(image_file) try: im = Image.open(os.path.join(dir_abs, image_file)) except IOError: "warning filename %s is not an image" % os.path.join( dir_abs, image_file) continue rgb = im.tobytes("raw", "RGB") a = im.tobytes("raw", "A") im.close() image = Image.frombytes("RGB", im.size, rgb) mask = Image.frombytes("L", im.size, a) image.save(os.path.join(pos_image_path, image_file), "PNG") mask.save(os.path.join(pos_mask_path, image_file), "PNG") else: print("Negative term") for image_file in os.listdir(dir_abs): print(image_file) try: im = Image.open(os.path.join(dir_abs, image_file)) except IOError: "warning filename %s is not an image" % os.path.join( dir_abs, image_file) continue rgb = im.tobytes("raw", "RGB") a = im.tobytes("raw", "A") im.close() image = Image.frombytes("RGB", im.size, rgb) mask = Image.frombytes("L", im.size, a) image.save(os.path.join(neg_image_path, image_file), "PNG") mask.save(os.path.join(neg_mask_path, image_file), "PNG") if parameters['cytomine_dump_annotation_stats']: pos_path = os.path.join(parameters['dir_ls'], "image", "1") neg_path = os.path.join(parameters['dir_ls'], "image", "0") stats_dumped_annotations(pos_path, neg_path) # if parameters['build_model'] : # # Model name # model_name = "nsubw{}_winsize{}x{}_minsize{}_maxsize{}_batchsize{}_epochs{}_shuffle{}_valsplit{}_colorspace{}"\ # .format(parameters['pyxit_n_subwindows'], # parameters['pyxit_target_width'], # parameters['pyxit_target_height'], # parameters['pyxit_min_size'], # parameters['pyxit_max_size'], # parameters['keras_batch_size'], # parameters['keras_n_epochs'], # parameters['keras_shuffle'], # parameters['keras_validation_split'], # pyxit_parameters['pyxit_colorspace']).replace(".", "") # print("Model_name :", model_name) # # pyxit = PyxitClassifier(None, # n_subwindows = pyxit_parameters['pyxit_n_subwindows'], # min_size = pyxit_parameters['pyxit_min_size'], # max_size = pyxit_parameters['pyxit_max_size'], # target_width = pyxit_parameters['pyxit_target_width'], # target_height = pyxit_parameters['pyxit_target_height'], # n_jobs = pyxit_parameters['pyxit_n_jobs'], # interpolation = pyxit_parameters['pyxit_interpolation'], # transpose = pyxit_parameters['pyxit_transpose'], # colorspace = pyxit_parameters['pyxit_colorspace'], # fixed_size = pyxit_parameters['pyxit_fixed_size'], # random_state = None, # verbose = 1, # get_output = _get_output_from_mask, # parallel_leaf_transform = False) # # # Build filenames and classes # X, y = build_from_dir(parameters['dir_ls']) # # classes = np.unique(y) # n_classes = len(classes) # y_original = y # y = np.searchsorted(classes, y) # n_images = len(y) # print("Number of images : ", n_images) # # # Extract subwindows # _X, _y = pyxit.extract_subwindows(X, y) # n_subw = len(_y) # print("Number of subwindows : ", n_subw) # # # Reshape data structure # _X = np.reshape(_X, ( # n_subw, pyxit_parameters['pyxit_target_width'], pyxit_parameters['pyxit_target_height'], n_channels)) # _y = np.reshape(_y, (n_subw, pyxit_parameters['pyxit_target_width'], pyxit_parameters['pyxit_target_height'])) # # # Train FCN # if not os.path.exists(parameters['keras_save_to']) : # os.makedirs(parameters['keras_save_to']) # # model_weights_file_path = os.path.join(parameters['keras_save_to'], "weights_" + model_name + ".h5") # # mean, std = train(_X, _y, # model_weights_file_path, # imgs_width = pyxit_parameters['pyxit_target_width'], # imgs_height = pyxit_parameters['pyxit_target_height'], # batch_size = parameters['keras_batch_size'], # epochs = parameters['keras_n_epochs'], # shuffle = parameters['keras_shuffle'], # validation_split = parameters['keras_validation_split']) # # # Save mean and std used to normalize training data # mean_std_save_file_path = os.path.join(parameters['keras_save_to'], "meanstd_" + model_name + ".txt") # mean_std_save_file = open(mean_std_save_file_path, 'w') # mean_std_save_file.write(str(mean) + '\n') # mean_std_save_file.write(str(std) + '\n') if parameters['build_model']: # Model name model_name = "nsubw{}_winsize{}x{}_minsize{}_maxsize{}_batchsize{}_epochs{}_shuffle{}_valsplit{}_colorspace{}_zoom{}_until4x4_IDG"\ .format(parameters['pyxit_n_subwindows'], parameters['pyxit_target_width'], parameters['pyxit_target_height'], parameters['pyxit_min_size'], parameters['pyxit_max_size'], parameters['keras_batch_size'], parameters['keras_n_epochs'], parameters['keras_shuffle'], parameters['keras_validation_split'], pyxit_parameters['pyxit_colorspace'], parameters['cytomine_zoom_level']).replace(".", "") print("Model_name :", model_name) pyxit = PyxitClassifier( None, n_subwindows=1, min_size=1, max_size=1, target_width=pyxit_parameters['pyxit_target_width'], target_height=pyxit_parameters['pyxit_target_height'], n_jobs=pyxit_parameters['pyxit_n_jobs'], interpolation=pyxit_parameters['pyxit_interpolation'], transpose=pyxit_parameters['pyxit_transpose'], colorspace=pyxit_parameters['pyxit_colorspace'], fixed_size=pyxit_parameters['pyxit_fixed_size'], random_state=None, verbose=1, get_output=_get_output_from_mask, parallel_leaf_transform=False) # pyxit = PyxitClassifier(None, # n_subwindows=pyxit_parameters['pyxit_n_subwindows'], # min_size=pyxit_parameters['pyxit_min_size'], # max_size=pyxit_parameters['pyxit_max_size'], # target_width=pyxit_parameters['pyxit_target_width'], # target_height=pyxit_parameters['pyxit_target_height'], # n_jobs=pyxit_parameters['pyxit_n_jobs'], # interpolation=pyxit_parameters['pyxit_interpolation'], # transpose=pyxit_parameters['pyxit_transpose'], # colorspace=pyxit_parameters['pyxit_colorspace'], # fixed_size=pyxit_parameters['pyxit_fixed_size'], # random_state=None, # verbose=1, # get_output = _get_output_from_mask, # parallel_leaf_transform=False) # Build filenames and classes X, y = build_from_dir(parameters['dir_ls']) classes = np.unique(y) n_classes = len(classes) y_original = y y = np.searchsorted(classes, y) n_images = len(y) print("Number of images : ", n_images) print("Start extraction of subwindows...") # Extract subwindows _X, _y = pyxit.extract_subwindows(X, y) print("Over") n_subw = len(_y) print("Number of subwindows : ", n_subw) # Reshape data structure _X = np.reshape(_X, (n_subw, pyxit_parameters['pyxit_target_width'], pyxit_parameters['pyxit_target_height'], n_channels)) _y = np.reshape(_y, (n_subw, pyxit_parameters['pyxit_target_width'], pyxit_parameters['pyxit_target_height'], 1)) print(type(_X)) print(type(_y)) # ImageDataGenerator : two instances with the same arguments print("Init data gen") data_gen_args = dict(rotation_range=180., width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.2, rescale=1 / 255, horizontal_flip=True, vertical_flip=True) # featurewise_center = True, # featurewise_std_normalization = True) image_datagen = ImageDataGenerator(**data_gen_args) mask_datagen = ImageDataGenerator(**data_gen_args) # Provide the same seed and keyword arguments to the fit and flow methods seed = 1 print("Fit image data generator (image)...") image_datagen.fit(_X[0:100], augment=True, seed=seed) print("Fit image data generator (mask)...") mask_datagen.fit(_y[0:100], augment=True, seed=seed) print('Flow on images...') # image_generator = image_datagen.flow(_X, labels, seed = seed, shuffle = False) image_generator = image_datagen.flow_from_directory( os.path.join(parameters['dir_ls'], "image"), class_mode=None, target_size=(128, 128), seed=seed) print('Flow on masks...') # mask_generator = mask_datagen.flow(_y, labels, seed = seed, shuffle = False) mask_generator = mask_datagen.flow_from_directory( os.path.join(parameters['dir_ls'], "mask"), class_mode=None, target_size=(128, 128), seed=seed) # combine generators into one which yields image and masks train_generator = combine_generator(image_generator, mask_generator) # Creating and compiling model if not os.path.exists(parameters['keras_save_to']): os.makedirs(parameters['keras_save_to']) model_weights_filename = os.path.join(parameters['keras_save_to'], "weights_" + model_name + ".h5") print('Fitting model...') model = get_unet(128, 128) model_checkpoint = ModelCheckpoint(model_weights_filename, monitor='val_loss', save_best_only=True) # Train FCN model.fit_generator(train_generator, steps_per_epoch=100, epochs=50, callbacks=[model_checkpoint], verbose=1)
def main(argv): # Define command line options p = optparse.OptionParser( description='Pyxit/Cytomine Classification model Builder', prog='PyXit Classification Model Builder (PYthon piXiT)') p.add_option( "--cytomine_host", type="string", default='', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('--cytomine_dump_type', type='int', dest='cytomine_dump_type', help="annotation type (1=crop, 2=alphamask)") # p.add_option('--cytomine_fixed_tile', default=False,action="store_true", dest="cytomine_fixed_tile", help="Force fixed tile size crop around annotations") p.add_option('--cytomine_fixed_tile', type="string", default="False", dest="cytomine_fixed_tile", help="Force fixed tile size crop around annotations") p.add_option( '--cytomine_n_shifts', type='int', dest='cytomine_n_shifts', help= "number of translated (shifted) crops extracted for each annotation") p.add_option('--cytomine_annotation_projects', type="string", dest="cytomine_annotation_projects", help="Projects from which annotations are extracted") p.add_option('--cytomine_excluded_terms', type='string', default='5735', dest='cytomine_excluded_terms', help="term ids of excluded terms") # p.add_option('--cytomine_reviewed', default=False, action="store_true", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--cytomine_reviewed', type='string', default="False", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--pyxit_target_width', type='int', dest='pyxit_target_width', help="pyxit subwindows width") p.add_option('--pyxit_target_height', type='int', dest='pyxit_target_height', help="pyxit subwindows height") p.add_option( '--pyxit_colorspace', type='int', dest='pyxit_colorspace', help="pyxit colorspace encoding") # future: get it from server db p.add_option( '--pyxit_n_jobs', type='int', dest='pyxit_n_jobs', help="pyxit number of jobs for trees") # future: get it from server db p.add_option('--pyxit_n_subwindows', default=10, type="int", dest="pyxit_n_subwindows", help="number of subwindows") p.add_option('--pyxit_min_size', default=0.5, type="float", dest="pyxit_min_size", help="min size") p.add_option('--pyxit_max_size', default=1.0, type="float", dest="pyxit_max_size", help="max size") p.add_option('--pyxit_interpolation', default=2, type="int", dest="pyxit_interpolation", help="interpolation method 1,2,3,4") # p.add_option('--pyxit_transpose', default=False, action="store_true", dest="pyxit_transpose", help="transpose subwindows") p.add_option('--pyxit_transpose', type="string", default="False", dest="pyxit_transpose", help="transpose subwindows") # p.add_option('--pyxit_fixed_size', default=False, action="store_true", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--pyxit_fixed_size', type="string", default="False", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--forest_n_estimators', default=10, type="int", dest="forest_n_estimators", help="number of base estimators (T)") p.add_option('--forest_max_features', default=1, type="int", dest="forest_max_features", help="max features at test node (k)") p.add_option('--forest_min_samples_split', default=1, type="int", dest="forest_min_samples_split", help="minimum node sample size (nmin)") p.add_option( '--svm', default=0, dest="svm", help= "final svm classifier: 0=nosvm, 1=libsvm, 2=liblinear, 3=lr-l1, 4=lr-l2", type="int") p.add_option('--svm_c', default=1.0, type="float", dest="svm_c", help="svm C") p.add_option('--cv_k_folds', default=False, type="int", dest="cv_k_folds", help="number of cross validation folds") # p.add_option('--cv_shuffle', default=False, action="store_true", dest="cv_shuffle", help="shuffle splits in cross validation") p.add_option('--cv_shuffle', type="string", default="False", dest="cv_shuffle", help="shuffle splits in cross validation") p.add_option('--cv_shuffle_test_fraction', default=0.3, type="float", dest="cv_shuffle_test_fraction", help="shuffle fraction in cross validation") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=argv) parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_annotation_projects'] = map( int, options.cytomine_annotation_projects.split(',')) parameters['cytomine_excluded_terms'] = map( int, options.cytomine_excluded_terms.split(',')) parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['cytomine_dump_type'] = options.cytomine_dump_type parameters['cytomine_fixed_tile'] = str2bool(options.cytomine_fixed_tile) parameters['cytomine_n_shifts'] = options.cytomine_n_shifts parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed) pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators pyxit_parameters['forest_max_features'] = options.forest_max_features pyxit_parameters[ 'forest_min_samples_split'] = options.forest_min_samples_split pyxit_parameters['svm'] = options.svm pyxit_parameters['svm_c'] = options.svm_c pyxit_parameters['cv_k_folds'] = options.cv_k_folds pyxit_parameters['cv_shuffle'] = str2bool(options.cv_shuffle) pyxit_parameters[ 'cv_shuffle_test_fraction'] = options.cv_shuffle_test_fraction pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs # Check for errors in the options if options.verbose: print "[pyxit.main] Options = ", options # Create JOB/USER/JOB conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) # Create a new userjob if connected as human user current_user = conn.get_current_user() run_by_user_job = False if current_user.algo == False: print "adduserJob..." user_job = conn.add_user_job(parameters['cytomine_id_software'], parameters['cytomine_id_project']) print "set_credentials..." conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) print "done" else: user_job = current_user print "Already running as userjob" run_by_user_job = True job = conn.get_job(user_job.job) # get annotation collection from Cytomine annotations = conn.get_annotations( id_project=parameters['cytomine_id_project'], reviewed_only=parameters['cytomine_reviewed']) # set output dir parameters pyxit_parameters['dir_ls'] = os.path.join( parameters["cytomine_working_path"], str(parameters['cytomine_annotation_projects']).replace( ',', '-').replace('[', '').replace(']', '').replace(' ', ''), "zoom_level", str(parameters['cytomine_zoom_level']), "dump_type", str(parameters['cytomine_dump_type'])) if not os.path.exists(pyxit_parameters['dir_ls']): print "Creating annotation directory: %s" % pyxit_parameters['dir_ls'] os.makedirs(pyxit_parameters['dir_ls']) time.sleep(2) # image dump type if (parameters['cytomine_dump_type'] == 1): annotation_get_func = Annotation.get_annotation_crop_url elif (parameters['cytomine_dump_type'] == 2): annotation_get_func = Annotation.get_annotation_alpha_crop_url else: print "default annotation type crop" annotation_get_func = Annotation.get_annotation_crop_url # dump annotation crops job = conn.update_job_status(job, status=job.RUNNING, status_comment="Dump annotations...", progress=50) annotations = conn.dump_annotations( annotations=annotations, get_image_url_func=annotation_get_func, dest_path=pyxit_parameters['dir_ls'], desired_zoom=parameters['cytomine_zoom_level'], excluded_terms=parameters['cytomine_excluded_terms'], tile_size=parameters['cytomine_fixed_tile'], translate=parameters['cytomine_n_shifts']) # build pyxit model(s) and evaluate them (according to cross-validation parameters) print "Create software parameters values..." if run_by_user_job == False: parameters_values = conn.add_job_parameters( user_job.job, conn.get_software(parameters['cytomine_id_software']), pyxit_parameters) argv = parameters_values_to_argv(pyxit_parameters, parameters_values) else: argv = [] for key in pyxit_parameters: value = pyxit_parameters[key] if type(value) is bool or value == 'True': if bool(value): argv.append("--%s" % key) elif not value == 'False': argv.append("--%s" % key) argv.append("%s" % value) print "Run PyXiT..." print argv job = conn.update_job_status(job, status=job.RUNNING, status_comment="Build models...", progress=75) predict = pyxitstandalone.main(argv) print "------------------- Publishing results to Cytomine Core... ----------------------" print annotations.data() job = conn.update_job_status(job, status=job.RUNNING, status_comment="Publishing results...", progress=90) for annotation in annotations.data(): # print "ANNOTATION: %s" %annotation # print "ANNOTATION TERM: %s" %annotation.term for term in annotation.term: # annot_descr = conn.get_annotation(annotation.id) # if hasattr(annotation, "filename"): # print "filename: %s" %annotation.filename # time.sleep(1) if hasattr(annotation, "filename") and (annotation.filename in predict): print "PUBLISH annotation %s prediction" % annotation p = predict[annotation.filename] annotation_term = AlgoAnnotationTerm() annotation_term.annotation = annotation.id annotation_term.expectedTerm = term annotation_term.term = p[0] if (pyxit_parameters['svm'] == 1 ): # libsvm does not return proba annotation_term.rate = 1.0 else: annotation_term.rate = p[1] conn.add_annotation_term( annotation.id, p[0], term, annotation_term.rate, annotation_term_model=AlgoAnnotationTerm) job = conn.update_job_status(job, status=job.TERMINATED, status_comment="Finish", progress=100) print "END."
def main(): opt_parser = optparse.OptionParser(description='Cytomine Landmark Detection : Model building', prog='Cytomine Landmark Detector : Model builder', version='0.1') opt_parser.add_option('--cytomine_host', type="string", default='beta.cytomine.be', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") opt_parser.add_option('--cytomine_public_key', type="string", default='XXX', dest="cytomine_public_key", help="Cytomine public key") opt_parser.add_option('--cytomine_private_key', type="string", default='YYY', dest="cytomine_private_key", help="Cytomine private key") opt_parser.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") opt_parser.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") opt_parser.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") opt_parser.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") opt_parser.add_option('--cytomine_id_terms', type='string', dest='cytomine_id_terms', help="The identifiers of the terms to create detection models for. Terms must be separated by commas (no spaces). If 'all' is mentioned instead, every terms will be detected.") opt_parser.add_option('--cytomine_training_images', default='all', type='string', dest='cytomine_training_images', help="identifiers of the images used to create the models. ids must be separated by commas (no spaces). If 'all' is mentioned instead, every image with manual annotation will be used.") opt_parser.add_option('--image_type', type='string', default='jpg', dest='image_type', help="The type of the images that will be used (jpg, bmp, png,...)") opt_parser.add_option('--model_njobs', type='int', default=1, dest='model_njobs', help="The number of processors used for model building") opt_parser.add_option('--model_R', type='int', default=6, dest='model_R', help="Max distance for extracting landmarks") opt_parser.add_option('--model_RMAX', type='int', default=200, dest='model_RMAX', help="Max distance for extracting non-landmarks") opt_parser.add_option('--model_P', type='float', default=3, dest='model_P', help="Proportion of non-landmarks") opt_parser.add_option('--model_npred', type='int', default=50000, dest='model_npred', help="Number of pixels extracted for prediction") opt_parser.add_option('--model_ntrees', type='int', default=50, dest='model_ntrees', help="Number of trees") opt_parser.add_option('--model_ntimes', type='int', default=3, dest='model_ntimes', help="Number of rotations to apply to the image") opt_parser.add_option('--model_angle', type='float', default=30, dest='model_angle', help="Max angle for rotation") opt_parser.add_option('--model_depth', type='int', default=5, dest='model_depth', help="Number of resolutions to use") opt_parser.add_option('--model_step', type='int', default=1, dest='model_step', help="Landmark pixels will be extracted in a grid (x-R:step:x+r,y-R:step:y+R) around the landmark") opt_parser.add_option('--model_wsize', type='int', default=8, dest='model_wsize', help="Window size") opt_parser.add_option('--model_feature_type', type='string', default='haar', dest='model_feature_type', help='The type of feature (raw, sub, haar or gaussian).') opt_parser.add_option('--model_feature_haar_n', type='int', default=1600, dest='model_feature_haar_n', help='Haar-Like features only. Number of descriptors for a pixel. Must be a multiple of 5*depths.') opt_parser.add_option('--model_feature_gaussian_n', type='int', default=1600, dest='model_feature_gaussian_n', help='Gaussian features only. Number of descriptors for a pixel. Must be a multiple of depths.') opt_parser.add_option('--model_feature_gaussian_std', type='float', default=20., dest='model_feature_gaussian_std', help='Gaussian features only. Standard deviation for the gaussian.') opt_parser.add_option('--model_save_to', type='string', default='/tmp/', dest='model_save_to', help="Destination for model storage") opt_parser.add_option('--model_name', type='string', dest='model_name', help="Name of the model (used for saving)") opt_parser.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") opts, argus = opt_parser.parse_args(args=sys.argv) conn = cytomine.Cytomine(opts.cytomine_host, opts.cytomine_public_key, opts.cytomine_private_key, base_path=opts.cytomine_base_path, working_path=opts.cytomine_working_path, verbose=str2bool(opts.verbose)) current_user = conn.get_current_user() if not current_user.algo: user_job = conn.add_user_job(opts.cytomine_id_software, opts.cytomine_id_project) conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) else: user_job = current_user job = conn.get_job(user_job.job) job = conn.update_job_status(job, status=job.RUNNING, progress=0, status_comment="Bulding model...") job_parameters = {'cytomine_id_terms': opts.cytomine_id_terms, 'cytomine_training_images': opts.cytomine_training_images, 'model_R': opts.model_R, 'model_njobs': opts.model_njobs, 'model_RMAX': opts.model_RMAX, 'model_P': opts.model_P, 'model_npred': opts.model_npred, 'model_ntimes': opts.model_ntimes, 'model_angle': opts.model_angle, 'model_depth': opts.model_depth, 'model_wsize': opts.model_wsize, 'model_ntrees': opts.model_ntrees, 'model_step': opts.model_step, 'forest_max_features': ((2 * opts.model_wsize) ** 2) * opts.model_depth, 'forest_min_samples_split': 2, 'model_name': opts.model_name, 'model_feature_type': opts.model_feature_type, 'model_feature_haar_n': opts.model_feature_haar_n, 'model_feature_gaussian_n': opts.model_feature_gaussian_n, 'model_feature_gaussian_std': opts.model_feature_gaussian_std} conn.add_job_parameters(user_job.job, conn.get_software(opts.cytomine_id_software), job_parameters) download_images(conn, opts.cytomine_id_project) download_annotations(conn, opts.cytomine_id_project, opts.cytomine_working_path) repository = opts.cytomine_working_path + str(opts.cytomine_id_project) + '/' txt_repository = opts.cytomine_working_path + '%d/txt/' % opts.cytomine_id_project depths = 1. / (2. ** np.arange(opts.model_depth)) (xc, yc, xr, yr, ims, t_to_i, i_to_t) = getallcoords(txt_repository) if opts.cytomine_id_terms == 'all': term_list = t_to_i.keys() else: term_list = [int(term) for term in opts.cytomine_id_terms.split(',')] if opts.cytomine_training_images == 'all': tr_im = ims else: tr_im = [int(id_im) for id_im in opts.cytomine_training_images.split(',')] DATA = None REP = None be = 0 model_repo = opts.model_save_to for id_term in term_list: (xc, yc, xr, yr) = getcoordsim(txt_repository, id_term, tr_im) nimages = np.max(xc.shape) mx = np.mean(xr) my = np.mean(yr) P = np.zeros((2, nimages)) P[0, :] = xr P[1, :] = yr cm = np.cov(P) passe = False progress = 0 delta = 80 / opts.model_ntimes # additional parameters feature_parameters = None if opts.model_feature_type.lower() == 'gaussian': std_matrix = np.eye(2) * (opts.model_feature_gaussian_std ** 2) feature_parameters = np.round(np.random.multivariate_normal([0, 0], std_matrix, opts.model_feature_gaussian_n)).astype(int) elif opts.model_feature_type.lower() == 'haar': W = opts.model_wsize n = opts.model_feature_haar_n / (5 * opts.model_depth) h2 = generate_2_horizontal(W, n) v2 = generate_2_vertical(W, n) h3 = generate_3_horizontal(W, n) v3 = generate_3_vertical(W, n) sq = generate_square(W, n) feature_parameters = (h2, v2, h3, v3, sq) for times in range(opts.model_ntimes): if times == 0: rangrange = 0 else: rangrange = opts.model_angle T = build_datasets_rot_mp(repository, tr_im, xc, yc, opts.model_R, opts.model_RMAX, opts.model_P, opts.model_step, rangrange, opts.model_wsize, opts.model_feature_type, feature_parameters, depths, nimages, opts.image_type, opts.model_njobs) for i in range(len(T)): (data, rep, img) = T[i] (height, width) = data.shape if not passe: passe = True DATA = np.zeros((height * (len(T) + 100) * opts.model_ntimes, width)) REP = np.zeros(height * (len(T) + 100) * opts.model_ntimes) b = 0 be = height DATA[b:be, :] = data REP[b:be] = rep b = be be = be + height progress += delta job = conn.update_job_status(job, status=job.RUNNING, status_comment="Bulding model...", progress=progress) REP = REP[0:b] DATA = DATA[0:b, :] clf = ExtraTreesClassifier(n_jobs=opts.model_njobs, n_estimators=opts.model_ntrees) clf = clf.fit(DATA, REP) job = conn.update_job_status(job, status=job.RUNNING, progress=90, status_comment="Writing model...") if not os.path.isdir(model_repo): os.mkdir(model_repo) joblib.dump(clf, '%s%s_%d.pkl' % (model_repo, opts.model_name, id_term)) joblib.dump([mx, my, cm], '%s%s_%d_cov.pkl' % (model_repo, opts.model_name, id_term)) if opts.model_feature_type == 'haar' or opts.model_feature_type == 'gaussian': joblib.dump(feature_parameters, '%s%s_%d_fparameters.pkl' % (model_repo, opts.model_name, id_term)) f = open('%s%s.conf' % (model_repo, opts.model_name), 'w+') f.write('cytomine_id_terms %s\n' % opts.cytomine_id_terms) f.write('model_R %d\n' % opts.model_R) f.write('model_RMAX %d\n' % opts.model_RMAX) f.write('model_P %f\n' % opts.model_P) f.write('model_npred %d\n' % opts.model_npred) f.write('model_ntrees %d\n' % opts.model_ntrees) f.write('model_ntimes %d\n' % opts.model_ntimes) f.write('model_angle %f\n' % opts.model_angle) f.write('model_depth %d\n' % opts.model_depth) f.write('model_step %d\n' % opts.model_step) f.write('window_size %d\n' % opts.model_wsize) f.write('feature_type %s\n' % opts.model_feature_type) f.write('feature_haar_n %d\n' % opts.model_feature_haar_n) f.write('feature_gaussian_n %d\n' % opts.model_feature_gaussian_n) f.write('feature_gaussian_std %f' % opts.model_feature_gaussian_std) f.close() conn.update_job_status(job, status=job.TERMINATED, progress=100, status_comment="Model built!")
def main(argv): # Define command line options p = optparse.OptionParser( description='Pyxit/Cytomine Segmentation Model Builder', prog='PyXit Segmentation Model Builder (PYthon piXiT)') p.add_option( "--cytomine_host", type="string", default='', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('--cytomine_annotation_projects', type="string", dest="cytomine_annotation_projects", help="Projects from which annotations are extracted") p.add_option( '--cytomine_predict_terms', type='string', default='0', dest='cytomine_predict_terms', help="term ids of predicted terms (=positive class in binary mode)") p.add_option('--cytomine_excluded_terms', type='string', default='0', dest='cytomine_excluded_terms', help="term ids of excluded terms") p.add_option('--cytomine_reviewed', type='string', default="False", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--pyxit_target_width', type='int', dest='pyxit_target_width', help="pyxit subwindows width") p.add_option('--pyxit_target_height', type='int', dest='pyxit_target_height', help="pyxit subwindows height") p.add_option('--pyxit_save_to', type='string', dest='pyxit_save_to', help="pyxit model directory") #future: get it from server db p.add_option( '--pyxit_colorspace', type='int', dest='pyxit_colorspace', help="pyxit colorspace encoding") #future: get it from server db p.add_option( '--pyxit_n_jobs', type='int', dest='pyxit_n_jobs', help="pyxit number of jobs for trees") #future: get it from server db p.add_option('--pyxit_n_subwindows', default=10, type="int", dest="pyxit_n_subwindows", help="number of subwindows") p.add_option('--pyxit_interpolation', default=2, type="int", dest="pyxit_interpolation", help="interpolation method 1,2,3,4") p.add_option('--pyxit_transpose', type="string", default="False", dest="pyxit_transpose", help="transpose subwindows") p.add_option('--pyxit_fixed_size', type="string", default="False", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--forest_n_estimators', default=10, type="int", dest="forest_n_estimators", help="number of base estimators (T)") p.add_option('--forest_max_features', default=1, type="int", dest="forest_max_features", help="max features at test node (k)") p.add_option('--forest_min_samples_split', default=1, type="int", dest="forest_min_samples_split", help="minimum node sample size (nmin)") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=argv) parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_annotation_projects'] = map( int, options.cytomine_annotation_projects.split(',')) parameters['cytomine_predict_terms'] = map( int, options.cytomine_predict_terms.split(',')) parameters['cytomine_excluded_terms'] = map( int, options.cytomine_excluded_terms.split(',')) parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed) pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators pyxit_parameters['forest_max_features'] = options.forest_max_features pyxit_parameters[ 'forest_min_samples_split'] = options.forest_min_samples_split pyxit_parameters['pyxit_save_to'] = options.pyxit_save_to pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs # Check for errors in the options if options.verbose: print "[pyxit.main] Options = ", options # Create JOB/USER/JOB conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) #Create a new userjob if connected as human user current_user = conn.get_current_user() if current_user.algo == False: print "adduserJob..." user_job = conn.add_user_job(parameters['cytomine_id_software'], parameters['cytomine_id_project']) print "set_credentials..." conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey)) print "done" else: user_job = current_user print "Already running as userjob" job = conn.get_job(user_job.job) pyxit_parameters['dir_ls'] = os.path.join( parameters["cytomine_working_path"], str(parameters['cytomine_annotation_projects']).replace( ',', '-').replace('[', '').replace(']', '').replace(' ', ''), "zoom_level", str(parameters['cytomine_zoom_level'])) if not os.path.exists(pyxit_parameters['dir_ls']): print "Creating annotation directory: %s" % pyxit_parameters['dir_ls'] os.makedirs(pyxit_parameters['dir_ls']) time.sleep(2) job = conn.update_job_status( job, status_comment="Publish software parameters values") all_params = pyxit_parameters all_params.update(parameters) job_parameters_values = conn.add_job_parameters( user_job.job, conn.get_software(parameters['cytomine_id_software']), all_params) #Get annotation data job = conn.update_job_status(job, status=job.RUNNING, status_comment="Fetching data", progress=0) #Retrieve annotations from each annotation projects, either reviewed or unreviewed annotations annotations = None for prj in parameters['cytomine_annotation_projects']: if parameters["cytomine_reviewed"]: print "Retrieving reviewed annotations..." annotations_prj = conn.get_annotations(id_project=prj, reviewed_only=True) print "Reviewed annotations: %d" % len(annotations_prj.data()) else: print "Retrieving (unreviewed) annotations..." annotations_prj = conn.get_annotations(id_project=prj) print "(Unreviewed) annotations: %d" % len(annotations_prj.data()) if not annotations: annotations = annotations_prj else: annotations.data().extend(annotations_prj.data()) print "Nb annotations so far... = %d" % len(annotations.data()) time.sleep(3) print "Total annotations projects %s = %d" % ( parameters['cytomine_annotation_projects'], len(annotations.data())) time.sleep(3) print "Predict terms / excluded terms" print parameters['cytomine_predict_terms'] print parameters['cytomine_excluded_terms'] time.sleep(3) annotations = conn.dump_annotations( annotations=annotations, get_image_url_func=Annotation.get_annotation_alpha_crop_url, dest_path=pyxit_parameters['dir_ls'], excluded_terms=parameters['cytomine_excluded_terms'], desired_zoom=parameters['cytomine_zoom_level']) #Build matrix (subwindows described by pixel values and output) for training project = conn.get_project(parameters['cytomine_id_project']) terms = conn.get_terms(project.ontology) map_classes = { } # build X, Y. Change initial problem into binary problem : "predict_terms" vs others for term in terms.data(): if term.id in parameters['cytomine_predict_terms']: map_classes[term.id] = 1 else: map_classes[term.id] = 0 print pyxit_parameters #Prepare image matrix X, y = build_from_dir(pyxit_parameters['dir_ls'], map_classes) print "X length: %d " % len(X) print "Y length: %d " % len(y) time.sleep(5) #classes = np.unique(y) classes = [0, 1] n_classes = len(classes) y_original = y y = np.searchsorted(classes, y) # Instantiate classifiers job = conn.update_job_status( job, status=job.RUNNING, status_comment="[pyxit.main] Initializing PyxitClassifier...", progress=25) forest = ExtraTreesClassifier( n_estimators=pyxit_parameters['forest_n_estimators'], max_features=pyxit_parameters['forest_max_features'], min_samples_split=pyxit_parameters['forest_min_samples_split'], n_jobs=pyxit_parameters['pyxit_n_jobs'], verbose=True) pyxit = PyxitClassifier( base_estimator=forest, n_subwindows=pyxit_parameters['pyxit_n_subwindows'], min_size=0.0, #segmentation use fixed-size subwindows max_size=1.0, #segmentation use fixed-size subwindows target_width=pyxit_parameters['pyxit_target_width'], target_height=pyxit_parameters['pyxit_target_height'], interpolation=pyxit_parameters['pyxit_interpolation'], transpose=pyxit_parameters['pyxit_transpose'], colorspace=pyxit_parameters['pyxit_colorspace'], fixed_size=pyxit_parameters['pyxit_fixed_size'], n_jobs=pyxit_parameters['pyxit_n_jobs'], verbose=True, get_output=_get_output_from_mask) if pyxit_parameters['pyxit_save_to']: d = os.path.dirname(pyxit_parameters['pyxit_save_to']) if not os.path.exists(d): os.makedirs(d) fd = open(pyxit_parameters['pyxit_save_to'], "wb") pickle.dump(classes, fd, protocol=pickle.HIGHEST_PROTOCOL) job = conn.update_job_status( job, status_comment= "[pyxit.main] Extracting %d subwindows from each image in %s" % (pyxit_parameters['pyxit_n_subwindows'], pyxit_parameters['dir_ls']), progress=50) time.sleep(3) #Extract random subwindows in dumped annotations _X, _y = pyxit.extract_subwindows(X, y) #Build pixel classifier job = conn.update_job_status( job, status_comment="[pyxit.main] Fitting Pyxit Segmentation Model on %s", progress=75) print "TIME : %s" % strftime("%Y-%m-%d %H:%M:%S", localtime()) start = time.time() pyxit.fit(X, y, _X=_X, _y=_y) end = time.time() print "Elapsed time FIT: %d s" % (end - start) print "TIME : %s" % strftime("%Y-%m-%d %H:%M:%S", localtime()) print "pyxit.base_estimator.n_classes_" print pyxit.base_estimator.n_classes_ print "pyxit.base_estimator.classes_" print pyxit.base_estimator.classes_ if options.verbose: print "----------------------------------------------------------------" print "[pyxit.main] Saving Pyxit Segmentation Model locally into %s" % pyxit_parameters[ 'pyxit_save_to'] print "----------------------------------------------------------------" #Save model on local disk if pyxit_parameters['pyxit_save_to']: pickle.dump(pyxit, fd, protocol=pickle.HIGHEST_PROTOCOL) if pyxit_parameters['pyxit_save_to']: fd.close() print "Not Publishing model in db.." #job_data = conn.add_job_data(job, "model", pyxit_parameters['pyxit_save_to']) job = conn.update_job_status(job, status=job.TERMINATED, status_comment="Finish", progress=100) print "END."
def main(argv): # Parameter values are set through command-line, see test-train.sh parameters = { 'cytomine_host': None, 'cytomine_public_key': None, 'cytomine_private_key': None, 'cytomine_base_path': None, 'cytomine_working_path': '/home/maree/tmp/cytomine/annotations/', 'cytomine_id_software': 1, 'cytomine_id_project': 1, 'cytomine_zoom_level': 1, 'cytomine_dump_type': 1, 'cytomine_annotation_projects': [1], # id of projets from which we dump annotations for learning 'cytomine_predict_terms': [1], # 'cytomine_excluded_terms': [2, 3], # exclude these term ids 'cytomine_reviewed': True } # Parameter values are set through command-line, see test-train.sh pyxit_parameters = { 'dir_ls': "/", # 'dir_ts' : "/", 'forest_shared_mem': False, # processing 'pyxit_n_jobs': 10, # subwindows extraction 'pyxit_n_subwindows': 100, 'pyxit_min_size': 0.1, 'pyxit_max_size': 1.0, 'pyxit_target_width': 24, # 24x24 'pyxit_target_height': 24, 'pyxit_interpolation': 1, 'pyxit_transpose': 1, # do we apply rotation/mirroring to subwindows (to enrich training set) 'pyxit_colorspace': 2, # which colorspace do we use ? 'pyxit_fixed_size': False, # do we extracted fixed sizes or random sizes (false) # classifier parameters 'forest_n_estimators': 10, # number of trees 'forest_max_features': 28, # number of attributes considered at each node 'forest_min_samples_split': 1, # nmin 'svm': 0, 'svm_c': 1.0, } # Define command line options p = optparse.OptionParser( description='Pyxit/Cytomine Classification Model Builder', prog='PyXit Classification Model Builder (PYthon piXiT)') # Cytomine arguments p.add_option( "--cytomine_host", type="string", default='', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('--cytomine_dump_type', type='int', dest='cytomine_dump_type', help="annotation type (1=crop, 2=alphamask)") p.add_option('--cytomine_annotation_projects', type="string", dest="cytomine_annotation_projects", help="Projects from which annotations are extracted") p.add_option( '--cytomine_predict_terms', type='string', default='0', dest='cytomine_predict_terms', help="term ids of predicted terms (=positive class in binary mode)") p.add_option('--cytomine_excluded_terms', type='string', default='0', dest='cytomine_excluded_terms', help="term ids of excluded terms") p.add_option('--cytomine_reviewed', type='string', default="False", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--path_to_project_info', type="string", default=None, dest="path_to_project_info", help="Path to file containing the project information") p.add_option('--cytomine_test_projects', type="string", dest="cytomine_test_projects", help="Projects used to test the classification models") p.add_option('--cytomine_n_test_images_per_project', type="int", dest="cytomine_n_test_images_per_project", help="Number of images to test on per project") p.add_option('--build_models', type="string", default="0", dest="build_models", help="Turn on (1) or off (0) model building") p.add_option('--test_models', type="string", default="0", dest="test_models", help="Turn on (1) or off (0) model testing") p.add_option('--cytomine_dump', type="string", default="0", dest="cytomine_dump", help="Turn on (1) or off (0) model annotation dump") p.add_option('--dump_test_annotations', type="string", default="0", dest="dump_test_annotations", help="Turn on (1) or off (0) test annotation dump") p.add_option('--plot_results', type="string", default="0", dest="plot_results", help="Turn on (1) or off (0) result plot") # Pyxit arguments p.add_option('--pyxit_target_width', type='int', dest='pyxit_target_width', help="pyxit subwindows width") p.add_option('--pyxit_target_height', type='int', dest='pyxit_target_height', help="pyxit subwindows height") p.add_option('--pyxit_save_to', type='string', dest='pyxit_save_to', help="pyxit model directory") # future: get it from server db p.add_option( '--pyxit_colorspace', type='int', dest='pyxit_colorspace', help="pyxit colorspace encoding") # future: get it from server db p.add_option( '--pyxit_n_jobs', type='int', dest='pyxit_n_jobs', help="pyxit number of jobs for trees") # future: get it from server db p.add_option('--pyxit_n_subwindows', default=10, type="int", dest="pyxit_n_subwindows", help="number of subwindows") p.add_option('--pyxit_min_size', default=0.5, type="float", dest="pyxit_min_size", help="min size") p.add_option('--pyxit_max_size', default=1.0, type="float", dest="pyxit_max_size", help="max size") p.add_option('--pyxit_interpolation', default=2, type="int", dest="pyxit_interpolation", help="interpolation method 1,2,3,4") p.add_option('--pyxit_transpose', type="string", default="False", dest="pyxit_transpose", help="transpose subwindows") p.add_option('--pyxit_fixed_size', type="string", default="False", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--cv_k_folds', type="int", dest="cv_k_folds", help="The number of folds") p.add_option('--forest_n_estimators', default=10, type="int", dest="forest_n_estimators", help="number of base estimators (T)") p.add_option('--forest_max_features', default=1, type="int", dest="forest_max_features", help="max features at test node (k)") p.add_option('--forest_min_samples_split', default=1, type="int", dest="forest_min_samples_split", help="minimum node sample size (nmin)") p.add_option('--forest_shared_mem', default=False, action="store_true", dest="forest_shared_mem", help="shared mem") p.add_option( '--svm', default=0, dest="svm", help= "final svm classifier: 0=nosvm, 1=libsvm, 2=liblinear, 3=lr-l1, 4=lr-l2", type="int") p.add_option('--svm_c', default=1.0, type="float", dest="svm_c", help="svm C") # p.add_option('--verbose', action="store_true", default=True, dest="verbose", help="Turn on verbose mode") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=argv) parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_annotation_projects'] = map( int, options.cytomine_annotation_projects.split(',')) parameters['cytomine_test_projects'] = map( int, options.cytomine_test_projects.split(',')) parameters['cytomine_predict_terms'] = map( int, options.cytomine_predict_terms.split(',')) parameters['cytomine_excluded_terms'] = map( int, options.cytomine_excluded_terms.split(',')) parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['cytomine_dump_type'] = options.cytomine_dump_type parameters['cytomine_dump'] = str2bool(options.cytomine_dump) parameters['test_models'] = str2bool(options.test_models) parameters['build_models'] = str2bool(options.build_models) parameters['dump_test_annotations'] = str2bool( options.dump_test_annotations) parameters['plot_results'] = str2bool(options.plot_results) parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed) parameters['path_to_project_info'] = options.path_to_project_info parameters[ 'cytomine_n_test_images_per_project'] = options.cytomine_n_test_images_per_project pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) pyxit_parameters['cv_k_folds'] = options.cv_k_folds pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators pyxit_parameters['forest_max_features'] = options.forest_max_features pyxit_parameters[ 'forest_min_samples_split'] = options.forest_min_samples_split pyxit_parameters['forest_shared_mem'] = options.forest_min_samples_split pyxit_parameters['svm'] = options.svm pyxit_parameters['svm_c'] = options.svm_c pyxit_parameters['pyxit_save_to'] = options.pyxit_save_to pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs pyxit_parameters['dir_ls'] = os.path.join( parameters["cytomine_working_path"], str(parameters['cytomine_annotation_projects']).replace( ',', '-').replace('[', '').replace(']', '').replace(' ', ''), "zoom_level", str(parameters['cytomine_zoom_level']), "dump_type", str(parameters['cytomine_dump_type'])) # Check for errors in the options if options.verbose: print "[pyxit.main] Options = ", options # Create JOB/USER/JOB conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) # Image dump type (for classification use 1) if (parameters['cytomine_dump_type'] == 1): annotation_get_func = Annotation.get_annotation_crop_url elif (parameters['cytomine_dump_type'] == 2): annotation_get_func = Annotation.get_annotation_alpha_crop_url else: print "default annotation type crop" annotation_get_func = Annotation.get_annotation_crop_url if parameters['cytomine_dump']: # Get annotation descriptions (JSON) from project(s) annotations = None for prj in parameters['cytomine_annotation_projects']: if parameters["cytomine_reviewed"]: print "Retrieving reviewed annotations..." annotations_prj = conn.get_annotations( id_project=prj, reviewed_only=parameters["cytomine_reviewed"]) print "Reviewed annotations: %d" % len(annotations_prj.data()) else: # We go here print "Retrieving (unreviewed) annotations..." annotations_prj = conn.get_annotations(id_project=prj) print "(Unreviewed) annotations: %d" % len( annotations_prj.data()) if not annotations: annotations = annotations_prj else: annotations.data().extend(annotations_prj.data()) if prj == 21907448 or prj == 155194683: annotations_prj = conn.get_annotations(id_project=prj, id_term=91376951) print "Notadeno annotations: %d" % len(annotations_prj.data()) annotations.data().extend(annotations_prj.data()) print "Nb annotations so far... = %d" % len(annotations.data()) print "Total annotations projects %s = %d" % ( parameters['cytomine_annotation_projects'], len( annotations.data())) # Set output dir parameters if not os.path.exists(pyxit_parameters['dir_ls']): print "Creating annotation directory: %s" % pyxit_parameters[ 'dir_ls'] os.makedirs(pyxit_parameters['dir_ls']) # Dump annotation images locally conn.dump_annotations( annotations=annotations, get_image_url_func=annotation_get_func, dest_path=pyxit_parameters['dir_ls'], desired_zoom=parameters['cytomine_zoom_level'], excluded_terms=parameters['cytomine_excluded_terms']) quit() # Put positive terms under the same term and same for negative terms term_directories = os.listdir(pyxit_parameters['dir_ls']) pos_path = os.path.join(pyxit_parameters['dir_ls'], "1") if not os.path.exists(pos_path): print "Creating positive annotation directory: %s" % pos_path os.makedirs(pos_path) neg_path = os.path.join(pyxit_parameters['dir_ls'], "0") if not os.path.exists(neg_path): print "Creating negative annotation directory: %s" % neg_path os.makedirs(neg_path) print parameters['cytomine_predict_terms'] for dir in term_directories: dir_abs = os.path.join(pyxit_parameters['dir_ls'], dir) # Move files if int(dir) in parameters['cytomine_predict_terms']: for image_file in os.listdir(dir_abs): os.rename(os.path.join(dir_abs, image_file), os.path.join(pos_path, image_file)) else: for image_file in os.listdir(dir_abs): os.rename(os.path.join(dir_abs, image_file), os.path.join(neg_path, image_file)) # Remove empty directory os.rmdir(dir_abs) print pyxit_parameters['dir_ls'] os.path.dirname(pyxit_parameters['dir_ls']) # Put negative terms undre the same term print "END dump annotations" # Build models if parameters['build_models']: # Create directory if pyxit_parameters['cv_k_folds'] is None: d = os.path.dirname(pyxit_parameters['pyxit_save_to']) if not os.path.exists(d): os.makedirs(d) # Build models with varying parameters min_size = 0.1 for max_size in np.arange(0.15, 0.8, 0.05): # Model without svm build_model(pyxit_parameters, parameters, min_size, max_size, n_trees=100, svm=0, min_samples_split=10) quit() # Model with svm build_model(pyxit_parameters, parameters, min_size, max_size, n_trees=100, svm=1, min_samples_split=100) # Dump test annotations if parameters['dump_test_annotations']: # Dump test project images image_folder_records = open("image_folders_record.csv", 'w') annotations = None for project_id in parameters['cytomine_test_projects']: # Get userjob_id to test for each image if parameters['path_to_project_info'] is not None: path = os.path.join(parameters["path_to_project_info"], str(project_id)) df = pd.read_csv(os.path.join(path, "image_info.csv"), sep=';') images = df.loc[df['Term ID'] == 20202][np.isfinite( df['Job user ID'])].as_matrix(['Image ID', 'Job user ID']) else: images = np.array([ parameters["cytomine_id_image"], parameters['cytomine_id_userjob'] ]) # Select random images from project n_images = len(images) n_test_images_per_project = parameters[ 'cytomine_n_test_images_per_project'] if n_images < parameters['cytomine_n_test_images_per_project']: n_test_images_per_project = n_images if n_images == 0 and n_test_images_per_project == 0: continue random_idx = np.random.choice(n_images, n_test_images_per_project, replace=False) # For each randomly selected image, for idx in random_idx: i_id = int(images[idx][0]) u_id = int(images[idx][1]) candidate_annotations = conn.get_annotations( id_user=u_id, id_image=i_id, id_project=project_id) print "Number of annotations to predict: %d" % len( candidate_annotations.data()) folder_name = os.path.join( parameters["cytomine_working_path"], "annotations", "project-{}".format(project_id), "crops-candidates-{}-{}".format(i_id, u_id), "zoom-{}".format(parameters['cytomine_zoom_level'])) if not os.path.exists(folder_name): os.makedirs(folder_name) print "Dumping annotation cropped images to classify to %s" % folder_name # Dump annotation images locally conn.dump_annotations( annotations=candidate_annotations, get_image_url_func=annotation_get_func, dest_path=folder_name, desired_zoom=parameters['cytomine_zoom_level']) # Record image in file image_folder_records.write(folder_name + "\n") image_folder_records.close() # Test each model with each test image if parameters['test_models']: # Open folder with where the annotation images are with open("image_folders_record.csv", 'r') as f: image_folders = f.readlines() model_folder = pyxit_parameters['pyxit_save_to'] print "Open model folder :" print model_folder for model_name in sorted(os.listdir(model_folder)): print "Open model name :" print model_name # Create model test result folder test_results_folder = os.path.join( parameters['cytomine_working_path'], "test_results_colorspace1", model_name) if not os.path.exists(test_results_folder): os.makedirs(test_results_folder) for model in sorted( os.listdir(os.path.join(model_folder, model_name))): # Record model test results test_result_file_name = model.strip('.pkl') + '.csv' test_result_file = open( os.path.join(test_results_folder, test_result_file_name), 'w') test_result_file.write("Project ID;" "Image ID;" "Userjob ID;" "Annotation ID;" "Reviewed;" "Prediction;" "Positive prediction probability;" "Negative prediction probability\n") # Load Classifier model print "Open model :" print model model_path = os.path.join(model_folder, model_name, model) classifier = open(model_path, "r") classes = pickle.load(classifier) pyxit = pickle.load(classifier) print "Model: %s" % pyxit # Retrieve model information annotation_projects, rest = model.split("_svm") tmp, rest = rest.split("_minsize") svm = int(tmp) tmp, rest = rest.split("%_maxsize") min_size = int(tmp) / 100 tmp, rest = rest.split("%_ntrees") max_size = int(tmp) / 100 tmp, rest = rest.split(".pkl") n_trees = int(tmp) for image_folder_name in image_folders: image_folder_name = image_folder_name.strip("\n") # Get image annotation information zoom = os.path.basename(image_folder_name).strip("zoom-") rest = os.path.dirname(image_folder_name) candidate_name = os.path.basename(rest).strip( "crops-candidates-") image_id, userjob_id = candidate_name.split("-") image_id = int(image_id) userjob_id = int(userjob_id) rest = os.path.dirname(rest) project_id = int(os.path.basename(rest).strip("project-")) # Get reviewed annotations from image terms = str(parameters['cytomine_predict_terms']).strip( '[').strip(']').replace(' ', '') annotations = conn.get_annotations(id_project=project_id, id_image=image_id, id_term=terms, showWKT=True, showMeta=True, showGIS=True, reviewed_only=True) # Get parent id of reviewed annotations (Annotations whose ID is a key in the table is reviewed) review_parent_id_table = {} for annot in annotations.data(): # Get parent id and put (parent_id, reviewed_id) in a table review_parent_id_table[annot.parentIdent] = annot.id print "Building subwindows from ", image_folder_name # Extract subwindows from all candidates annotations X, y = pyxitstandalone.build_from_dir(image_folder_name) # Apply pyxit classifier model to X (parameters are already reused from model pkl file) y_proba = pyxit.predict_proba(X) * 100 for j, annotation_file_name in enumerate(X): print annotation_file_name annotation_id = int( os.path.basename(annotation_file_name).strip( '.png').split('_')[1]) y_predict = classes.take(np.argmax(y_proba[j], 0), axis=0) y_rate = np.max(y_proba[j]) # Check if annotation has been reviewed reviewed = False if review_parent_id_table.has_key(annotation_id): reviewed = True test_result_file.write( "{};{};{};{};{};{};{};{}\n".format( project_id, image_id, userjob_id, annotation_id, reviewed, y_predict, y_proba[j, 1], y_proba[j, 0])) test_result_file.close() # Plot results if parameters['plot_results']: test_results_folder = os.path.join(parameters['cytomine_working_path'], "test_results_colorspace0") stats_file = open( os.path.join(parameters['cytomine_working_path'], 'stats.txt'), 'w') # file containing test stats for sub_folder in sorted(os.listdir(test_results_folder)): for results_file in sorted( os.listdir(os.path.join(test_results_folder, sub_folder))): # Read results file print os.path.join(test_results_folder, sub_folder, results_file) # Read csv df = pd.read_csv(os.path.join(test_results_folder, sub_folder, results_file), sep=';') df.insert(0, "Index", df.index) n_data = df.shape[0] model = os.path.join(sub_folder, results_file.strip('.csv') + '.plk') stats_file.write('\n\nModel name : {}\n'.format(model)) # Find unique project ids plt.figure(figsize=[14, 7]) unique_project_ids = df['Project ID'].unique() n_projects = len(unique_project_ids) # Get annotation prediciton probability pred_positive = df.loc[df['Reviewed'] == True].as_matrix( ["Index", "Positive prediction probability"]) pred_negative = df.loc[df['Reviewed'] == False].as_matrix( ["Index", "Positive prediction probability"]) precision_array = np.zeros(n_projects) recall_array = np.zeros(n_projects) tp_array = np.zeros(n_projects) fn_array = np.zeros(n_projects) fp_array = np.zeros(n_projects) tn_array = np.zeros(n_projects) for i, id in enumerate(unique_project_ids): project_data = df.loc[df['Project ID'] == id] idx_id = int(project_data['Index'].head(1)) plt.axvline(x=idx_id, color='b', linestyle='dashed', linewidth=1, alpha=0.3) # Write stats about model tests project_name = conn.get_project(id).name print project_name tp_array[i] = project_data.loc[ project_data['Reviewed'] == True].loc[ project_data['Prediction'] == 1].shape[0] fn_array[i] = project_data.loc[ project_data['Reviewed'] == True].loc[ project_data['Prediction'] == 0].shape[0] fp_array[i] = project_data.loc[ project_data['Reviewed'] == False].loc[ project_data['Prediction'] == 1].shape[0] tn_array[i] = project_data.loc[ project_data['Reviewed'] == False].loc[ project_data['Prediction'] == 0].shape[0] precision_array[i] = float(tp_array[i]) / ( float(tp_array[i]) + float(fp_array[i])) recall_array[i] = float(tp_array[i]) / ( float(tp_array[i]) + float(fn_array[i])) stats_file.write('\n\t{} ({})\n'.format(project_name, id)) stats_file.write('\tNumber annotations : {}\n'.format( project_data.shape[0])) stats_file.write('\tTP : {} \tFN : {}\t| {}\n'.format( tp_array[i], fn_array[i], (tp_array[i] + fn_array[i]))) stats_file.write('\tFP : {} \tTN : {}\t| {}\n'.format( fp_array[i], tn_array[i], (fp_array[i] + tn_array[i]))) stats_file.write('\t -----------------------\n') stats_file.write('\t {}\t\t{}\n'.format( (tp_array[i] + fp_array[i]), (fn_array[i] + tn_array[i]))) stats_file.write('\tPrecision : \t{}\n'.format( precision_array[i])) stats_file.write('\tRecall : \t{}\n'.format( recall_array[i])) # Plot : For each different model, scatter plot of all annotations of all projects plt.scatter(pred_negative[:, 0], pred_negative[:, 1], c='r', label="Actual negative", alpha=0.6) plt.scatter(pred_positive[:, 0], pred_positive[:, 1], c='g', label="Actual positive", alpha=0.7) plt.title('Classification probability of annotations') plt.xlabel('Annotations') plt.ylabel('Probability (%)') plt.ylim([-5, 105]) plt.xlim([-100, n_data + 100]) legend = plt.legend(loc='lower right', framealpha=0.1) # Save plot figure fig_name = model.strip('.pkl') + '.png' fig_path = os.path.join(parameters['cytomine_working_path'], "plots_colorspace0", fig_name) if not os.path.exists(os.path.dirname(fig_path)): os.makedirs(os.path.dirname(fig_path)) print fig_path plt.savefig(fig_path) plt.clf() plt.close() # Plot precision recall map for project : Each point is a project plt.figure() plt.scatter(recall_array * 100, precision_array * 100, c='b') plt.title('Precision-Recall map') plt.xlabel('Recall') plt.ylabel('Precision') plt.xlim([-5, 105]) plt.ylim([-5, 105]) plt.savefig( os.path.join(parameters['cytomine_working_path'], "plots_colorspace0", os.path.dirname(fig_name), "PR_" + os.path.basename(fig_name))) plt.clf() plt.close()
__author__ = "Vandaele Rémy <*****@*****.**>" __contributors__ = ["Marée Raphaël <*****@*****.**>"] __copyright__ = "Copyright 2010-2015 University of Liège, Belgium, http://www.cytomine.be/" import cytomine # connect to cytomine : parameters to set cytomine_host = "" cytomine_public_key = "" cytomine_private_key = "" id_project = 0 # Connection to Cytomine Core conn = cytomine.Cytomine(cytomine_host, cytomine_public_key, cytomine_private_key, base_path='/api/', working_path='/tmp/', verbose=True) # define software parameter template software = conn.add_software("Landmark_Model_Validation", "pyxitSuggestedTermJobService", "ValidateAnnotation") conn.add_software_parameter("landmark_term", software.id, "Number", None, True, 1, False) conn.add_software_parameter("landmark_r", software.id, "Number", None, True, 2, False) conn.add_software_parameter("landmark_rmax", software.id, "Number", None, True, 3, False) conn.add_software_parameter("landmark_p", software.id, "Number", None, True, 4,
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='XXX', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='YYY', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") options, arguments = p.parse_args(args=sys.argv) parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['verbose'] = str2bool(options.verbose) cytomine_connection = cytomine.Cytomine(parameters['cytomine_host'], parameters['cytomine_public_key'], parameters['cytomine_private_key'], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=parameters['verbose']) download_images(cytomine_connection, parameters['cytomine_id_project']) download_annotations(cytomine_connection, parameters['cytomine_id_project'], parameters['cytomine_working_path'])
F.write('%d %d\n' % (xpos[(id_terms[i], image.id)], ypos[(id_terms[i], image.id)])) F.close() return xpos, ypos if __name__ == "__main__": public_key = 'c3090afd-b793-4ae2-a5f3-2cfdaee01f32' private_key = 'd7768538-db66-4aa6-882c-55f520f1be69' wpath = sys.argv[1] if (wpath.endswith('/')): wpath = wpath + '/' cytomine_connection = cytomine.Cytomine('demo.cytomine.be', public_key, private_key, base_path='/api/', working_path=wpath, verbose=True) droso_terms = [ 6579647, 6581077, 6581992, 6583116, 6584107, 6585116, 6586002, 6587114, 6587962, 6588763, 6589668, 6590562, 6591526, 6592482, 6593390 ] cepha_terms = [ 6625929, 6626956, 6628031, 6628982, 6630085, 6630930, 6632153, 6633169, 6634164, 6635158, 6636231, 6637186, 6638098, 6638869, 6639680, 6640638, 6641592, 6641602, 6641610 ] zebra_terms = [ 6555577, 6555589, 6555603, 6555613, 6555621, 6555631, 6555631, 6555647, 6555657, 6555665, 6555675, 6555681, 6555691, 6555699, 6555709, 6555717,
def main(argv): print("Main") # Define command line options p = optparse.OptionParser(description='Cytomine Segmentation prediction', prog='Cytomine segmentation prediction', version='0.1') p.add_option( '--cytomine_host', type="string", default='beta.cytomine.be', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)") p.add_option('--cytomine_public_key', type="string", default='', dest="cytomine_public_key", help="Cytomine public key") p.add_option('--cytomine_private_key', type="string", default='', dest="cytomine_private_key", help="Cytomine private key") p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path") p.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier") p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)") p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier") p.add_option('-i', '--cytomine_id_image', type='int', dest='cytomine_id_image', help="image id from cytomine", metavar='IMAGE') p.add_option('-z', '--cytomine_zoom_level', type='int', dest='cytomine_zoom_level', help="working zoom level") p.add_option('-j', '--nb_jobs', type='int', dest='nb_jobs', help="number of parallel jobs") p.add_option( '--cytomine_predict_terms', type='str', dest='cytomine_predict_terms', help= "term id of all positive terms. The first term is the output predicted annotation term" ) p.add_option('--cytomine_excluded_terms', type='string', dest='cytomine_excluded_terms', help="term id of excluded terms)") p.add_option('--pyxit_target_width', type='int', dest='pyxit_target_width', help="pyxit subwindows width") p.add_option('--pyxit_target_height', type='int', dest='pyxit_target_height', help="pyxit subwindows height") p.add_option('--pyxit_colorspace', type='int', dest='pyxit_colorspace', help="pyxit colorspace encoding") p.add_option('--pyxit_nb_jobs', type='int', dest='pyxit_nb_jobs', help="pyxit number of jobs for trees") p.add_option('--pyxit_fixed_size', type='string', default="0", dest="pyxit_fixed_size", help="extract fixed size subwindows") p.add_option('--pyxit_transpose', type='string', default="0", dest="pyxit_transpose", help="transpose subwindows") p.add_option('--pyxit_n_subwindows', type='int', default="10", dest="pyxit_n_subwindows", help="number of subwindows") p.add_option('--pyxit_interpolation', default=2, type="int", dest="pyxit_interpolation", help="interpolation method 1,2,3,4") p.add_option('--pyxit_min_size', default=0.5, type="float", dest="pyxit_min_size", help="min size") p.add_option('--pyxit_max_size', default=1.0, type="float", dest="pyxit_max_size", help="max size") p.add_option('--cytomine_reviewed', type='string', default="False", dest="cytomine_reviewed", help="Get reviewed annotations only") p.add_option('--cytomine_dump_annotations', type='string', default="0", dest="cytomine_dump_annotations", help="Dump training annotations or not") p.add_option('--cytomine_dump_annotation_stats', type='string', default="0", dest="cytomine_dump_annotation_stats", help="Calculate stats on dumped annotations or not") p.add_option('--build_model', type="string", default="0", dest="build_model", help="Turn on (1) or off (0) model building") p.add_option('--cytomine_annotation_projects', type="string", dest="cytomine_annotation_projects", help="Projects from which annotations are extracted") p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode") p.add_option('--keras_save_to', type='string', default="", dest='keras_save_to', help="keras model weight file") p.add_option('--keras_batch_size', type="int", dest="keras_batch_size", help="Training batch size") p.add_option('--keras_n_epochs', type="int", dest="keras_n_epochs", help="Number of epochs") p.add_option('--keras_shuffle', type="string", dest="keras_shuffle", help="Turn on (1) or off (0) batch shuffle") p.add_option('--keras_validation_split', type="float", dest="keras_validation_split", help="Batch validation split") options, arguments = p.parse_args(args=argv) parameters = {} parameters['keras_save_to'] = options.keras_save_to parameters['keras_batch_size'] = options.keras_batch_size parameters['keras_n_epochs'] = options.keras_n_epochs parameters['keras_shuffle'] = options.keras_shuffle parameters['keras_validation_split'] = options.keras_validation_split parameters['cytomine_host'] = options.cytomine_host parameters['cytomine_public_key'] = options.cytomine_public_key parameters['cytomine_private_key'] = options.cytomine_private_key parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_working_path'] = options.cytomine_working_path parameters['cytomine_base_path'] = options.cytomine_base_path parameters['cytomine_id_project'] = options.cytomine_id_project parameters['cytomine_id_software'] = options.cytomine_id_software parameters['cytomine_predict_terms'] = map( int, options.cytomine_predict_terms.split(',')) parameters['cytomine_predicted_annotation_term'] = parameters[ 'cytomine_predict_terms'][0] parameters['cytomine_excluded_terms'] = map( int, options.cytomine_excluded_terms.split(',')) parameters['pyxit_colorspace'] = options.pyxit_colorspace parameters['pyxit_nb_jobs'] = options.pyxit_nb_jobs parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs parameters['cytomine_nb_jobs'] = options.pyxit_nb_jobs parameters['cytomine_id_image'] = options.cytomine_id_image parameters['cytomine_zoom_level'] = options.cytomine_zoom_level parameters['nb_jobs'] = options.nb_jobs parameters['pyxit_target_width'] = options.pyxit_target_width parameters['pyxit_target_height'] = options.pyxit_target_height parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows parameters['pyxit_interpolation'] = options.pyxit_interpolation parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) parameters['pyxit_min_size'] = options.pyxit_min_size parameters['pyxit_max_size'] = options.pyxit_max_size parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) parameters['cytomine_annotation_projects'] = map( int, options.cytomine_annotation_projects.split(',')) parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed) parameters['cytomine_dump_annotation_stats'] = str2bool( options.cytomine_dump_annotation_stats) parameters['cytomine_dump_annotations'] = str2bool( options.cytomine_dump_annotations) parameters['build_model'] = str2bool(options.build_model) parameters['dir_ls'] = os.path.join( parameters["cytomine_working_path"], str(parameters['cytomine_annotation_projects']).replace( ',', '-').replace('[', '').replace(']', '').replace(' ', ''), "zoom_level", str(parameters['cytomine_zoom_level'])) pyxit_parameters = {} pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose) pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size) pyxit_parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs if options.verbose: print(parameters) # Create Cytomine connection conn = cytomine.Cytomine(parameters["cytomine_host"], parameters["cytomine_public_key"], parameters["cytomine_private_key"], base_path=parameters['cytomine_base_path'], working_path=parameters['cytomine_working_path'], verbose=str2bool(options.verbose)) # Dump annotations if parameters['cytomine_dump_annotations']: # Get annotation descriptions (JSON) from project(s) annotations = None for prj in parameters['cytomine_annotation_projects']: if parameters["cytomine_reviewed"]: annotations_prj = conn.get_annotations( id_project=prj, reviewed_only=parameters["cytomine_reviewed"]) else: annotations_prj = conn.get_annotations(id_project=prj) if not annotations: annotations = annotations_prj else: annotations.data().extend(annotations_prj.data()) if prj == 21907448 or prj == 155194683: annotations_prj = conn.get_annotations(id_project=prj, id_term=91376951) annotations.data().extend(annotations_prj.data()) print("Nb annotations so far... = %d" % len(annotations.data())) print("Total annotations projects %s = %d" % (parameters['cytomine_annotation_projects'], len(annotations.data()))) # Set output dir parameters if not os.path.exists(parameters['dir_ls']): print("Creating annotation directory: %s" % parameters['dir_ls']) os.makedirs(parameters['dir_ls']) # Dump annotation images locally print("Dump training annotation images in %s...", parameters['dir_ls']) conn.dump_annotations( annotations=annotations, get_image_url_func=Annotation.get_annotation_alpha_crop_url, dest_path=parameters['dir_ls'], desired_zoom=parameters['cytomine_zoom_level'], excluded_terms=parameters['cytomine_excluded_terms']) # Put positive terms under the same term and same for negative terms term_directories = os.listdir(parameters['dir_ls']) pos_path = os.path.join(parameters['dir_ls'], "1") if not os.path.exists(pos_path): print("Creating positive annotation directory: %s" % pos_path) os.makedirs(pos_path) neg_path = os.path.join(parameters['dir_ls'], "0") if not os.path.exists(neg_path): print("Creating negative annotation directory: %s" % neg_path) os.makedirs(neg_path) for dir in term_directories: dir_abs = os.path.join(parameters['dir_ls'], dir) # Move files if int(dir) in parameters['cytomine_predict_terms']: for image_file in os.listdir(dir_abs): os.rename(os.path.join(dir_abs, image_file), os.path.join(pos_path, image_file)) else: for image_file in os.listdir(dir_abs): os.rename(os.path.join(dir_abs, image_file), os.path.join(neg_path, image_file)) # Remove empty directory if int(dir) != 0 and int(dir) != 1: os.rmdir(dir_abs) if parameters['cytomine_dump_annotation_stats']: pos_path = os.path.join(parameters['dir_ls'], "1") neg_path = os.path.join(parameters['dir_ls'], "0") stats_dumped_annotations(pos_path, neg_path) if parameters['build_model']: print("Build_model...") # Model name model_name = "all_in_batchsize{}_epochs{}"\ .format(parameters['keras_batch_size'], parameters['keras_n_epochs']).replace(".", "") print("Model_name :", model_name) pyxit = PyxitClassifier( None, n_subwindows=pyxit_parameters['pyxit_n_subwindows'], min_size=pyxit_parameters['pyxit_min_size'], max_size=pyxit_parameters['pyxit_max_size'], target_width=pyxit_parameters['pyxit_target_width'], target_height=pyxit_parameters['pyxit_target_height'], n_jobs=pyxit_parameters['pyxit_n_jobs'], interpolation=pyxit_parameters['pyxit_interpolation'], transpose=pyxit_parameters['pyxit_transpose'], colorspace=pyxit_parameters['pyxit_colorspace'], fixed_size=pyxit_parameters['pyxit_fixed_size'], random_state=None, verbose=1, get_output=_get_output_from_mask, parallel_leaf_transform=False) # Build filenames and classes X, y = build_from_dir(parameters['dir_ls']) classes = np.unique(y) n_classes = len(classes) y_original = y y = np.searchsorted(classes, y) n_images = len(y) print("Number of images : ", n_images) images, masks, labels = image_mask_builder( X, y, parameters['pyxit_colorspace']) # ImageDataGenerator : two instances with the same arguments data_gen_args = dict(rotation_range=180., width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.2, rescale=1 / 255, horizontal_flip=True, vertical_flip=True) # featurewise_center = True, # featurewise_std_normalization = True) image_datagen = ImageDataGenerator(**data_gen_args) mask_datagen = ImageDataGenerator(**data_gen_args) # Provide the same seed and keyword arguments to the fit and flow methods seed = 1 # image_datagen.fit(images, augment = True, seed = seed) # mask_datagen.fit(masks, augment = True, seed = seed) print(type(images)) print(type(masks)) print(type(labels)) print(images[0:10]) print(masks[0:10]) print(labels[0:10]) image_generator = image_datagen.flow(images, labels, seed=seed, shuffle=False) mask_generator = mask_datagen.flow(masks, labels, seed=seed, shuffle=False) # combine generators into one which yields image and masks train_generator = zip(image_generator, mask_generator) # Creating and compiling model if not os.path.exists(parameters['keras_save_to']): os.makedirs(parameters['keras_save_to']) model_weights_filename = os.path.join(parameters['keras_save_to'], "weights_" + model_name + ".h5") print('Fitting model...') model = get_unet() model_checkpoint = ModelCheckpoint(model_weights_filename, monitor='val_loss', save_best_only=True) # Train FCN model.fit_generator(train_generator, steps_per_epoch=100, epochs=30, callbacks=[model_checkpoint], verbose=1)