def add_software(cytomine_host, cytomine_public_key, cytomine_private_key, id_project=None):
#Connection to Cytomine Core
conn = cytomine.Cytomine(cytomine_host, cytomine_public_key, cytomine_private_key, base_path = '/api/', working_path = '/tmp/', verbose= True)
execute_command = "python algo/W_SpotDetection-Icy/wrapper.py --icy_scale3sensitivity $icy_scale3sensitivity " \
+ "--cytomine_host $host " \
+ "--cytomine_public_key $publicKey " \
+ "--cytomine_private_key $privateKey " \
+ "--cytomine_id_project $cytomine_id_project "
#define software parameter template
software = conn.add_software("W_SpotDetection-Icy", "createRabbitJobWithArgsService","ValidateAnnotation", execute_command)
conn.add_software_parameter("icy_scale3sensitivity", software.id, "Number", 40, True, 10, False)
#for logging (set by server)
conn.add_software_parameter("cytomine_id_software", software.id, "Number",0, True, 400, True)
conn.add_software_parameter("cytomine_id_project", software.id, "Number", 0, True, 500, True)
if id_project:
#add software to a given project
addSoftwareProject = conn.add_software_project(id_project,software.id)
from cytomine.models import SoftwareParameter, SoftwareParameterCollection
if __name__ == "__main__":
import cytomine
# Connect to cytomine, edit connection values
cytomine_host = "demo.cytomine.be"
cytomine_public_key = "274db678-5c0c-4497-beb6-6602a532aab8" # to complete
cytomine_private_key = "4e2e1047-c706-47ad-8ba1-95aae6dabbf0" # to complete
id_project = -1 # to complete
# Connection to Cytomine Core
conn = cytomine.Cytomine(cytomine_host,
cytomine_public_key,
cytomine_private_key,
working_path=os.path.join(tempfile.gettempdir(),
"cytomine"),
verbose=True)
# define software parameter template
software = conn.add_software("Demo_SLDC_Workflow_With_Pyxit",
"pyxitSuggestedTermJobService",
"ValidateAnnotation")
conn.add_software_parameter("cytomine_id_software", software.id, "Number",
0, True, 1, True)
conn.add_software_parameter("cytomine_id_project", software.id, "Number",
0, True, 100, True)
conn.add_software_parameter("cytomine_id_image", software.id, "Number", 0,
True, 200, True)
conn.add_software_parameter("n_jobs", software.id, "Number", 1, True, 300,
False)
def main():
p = optparse.OptionParser(
description='Cytomine Landmark Detection : Model building',
prog='Cytomine Landmark Detector : Model builder',
version='0.1')
p.add_option(
'--cytomine_host',
type="string",
default='beta.cytomine.be',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='XXX',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='YYY',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option(
'--cytomine_training_images',
type='string',
default='all',
dest='cytomine_training_images',
help=
"IDs of the training images. Must be separated by commas, no spaces. 'all' takes all the available annotated images."
)
p.add_option(
'--cytomine_id_terms',
type='string',
default=1,
dest='cytomine_id_terms',
help=
"The identifiers of the terms to create detection models for. Terms must be separated by commas (no spaces). If 'all' is mentioned instead, every terms will be detected."
)
p.add_option(
'--image_type',
type='string',
default='jpg',
dest='image_type',
help="The type of the images that will be used (jpg, bmp, png,...)")
p.add_option('--model_njobs',
type='int',
default=4,
dest='model_njobs',
help="The number of processors used for model building")
p.add_option('--model_D_MAX',
type='int',
default=6,
dest='model_D_MAX',
help="D_MAX parameter.")
p.add_option('--model_n_samples',
type='int',
default=200,
dest='model_n_samples',
help="Number of samples for phase 1.")
p.add_option('--model_W',
type='int',
default=100,
dest='model_W',
help="Window size for feature extraction.")
p.add_option('--model_n',
type='int',
default=20,
dest='model_n',
help="Number of samples extracted.")
p.add_option('--model_T',
type='int',
default=50,
dest='model_T',
help="Number of trees for phase 1.")
p.add_option('--model_step',
type='int',
default=3,
dest='model_step',
help="Step for prediction for phase 1.")
p.add_option('--model_n_reduc',
type='int',
default=2,
dest='model_n_reduc',
help="Size for PCA reduction in phase 2.")
p.add_option('--model_R_MAX',
type='int',
default=20,
dest='model_R_MAX',
help="Maximal radius for phase 2.")
p.add_option('--model_R_MIN',
type='int',
default=3,
dest='model_R_MIN',
help="Minimal radius for phase 2.")
p.add_option('--model_alpha',
type='float',
default=0.5,
dest='model_alpha',
help="Radius reduction parameter for phase 2.")
p.add_option('--model_save_to',
type='string',
default='/tmp/',
dest='model_save_to',
help="Destination for model storage")
p.add_option('--model_name',
type='string',
dest='model_name',
help="Name of the model (used for saving)")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=sys.argv)
options.cytomine_working_path = options.cytomine_working_path.rstrip(
'/') + '/'
cytomine_connection = cytomine.Cytomine(
options.cytomine_host,
options.cytomine_public_key,
options.cytomine_private_key,
base_path=options.cytomine_base_path,
working_path=options.cytomine_working_path,
verbose=options.verbose)
current_user = cytomine_connection.get_current_user()
if not current_user.algo:
user_job = cytomine_connection.add_user_job(
options.cytomine_id_software, options.cytomine_id_project)
cytomine_connection.set_credentials(str(user_job.publicKey),
str(user_job.privateKey))
else:
user_job = current_user
job = cytomine_connection.get_job(user_job.job)
cytomine_connection.update_job_status(job,
status=job.RUNNING,
progress=0,
status_comment="Bulding model...")
model_repo = options.model_save_to
if not os.path.isdir(model_repo):
os.mkdir(model_repo)
download_images(cytomine_connection, options.cytomine_id_project)
download_annotations(cytomine_connection, options.cytomine_id_project,
options.cytomine_working_path)
repository = options.cytomine_working_path + str(
options.cytomine_id_project) + '/'
(xc, yc, xr, yr, ims, term_to_i,
i_to_term) = getallcoords(repository.rstrip('/') + '/txt/')
(nims, nldms) = xc.shape
if options.cytomine_id_terms != 'all':
term_list = [
int(term) for term in options.cytomine_id_terms.split(',')
]
Xc = np.zeros((nims, len(term_list)))
Yc = np.zeros(Xc.shape)
i = 0
for id_term in term_list:
Xc[:, i] = xc[:, term_to_i[id_term]]
Yc[:, i] = yc[:, term_to_i[id_term]]
i += 1
else:
term_list = term_to_i.keys()
Xc = xc
Yc = yc
(nims, nldms) = Xc.shape
if options.cytomine_training_images != 'all':
im_list = [int(p) for p in options.cytomine_training_images.split(',')]
else:
im_list = ims
X = np.zeros((len(im_list), nldms))
Y = np.zeros((len(im_list), nldms))
im_to_i = {}
for i in range(nims):
im_to_i[ims[i]] = i
for i in range(len(im_list)):
X[i, :] = Xc[im_to_i[im_list[i]], :]
Y[i, :] = Yc[im_to_i[im_list[i]], :]
Xc = X
Yc = Y
h2 = generate_2_horizontal(options.model_W, options.model_n)
v2 = generate_2_vertical(options.model_W, options.model_n)
h3 = generate_3_horizontal(options.model_W, options.model_n)
v3 = generate_3_vertical(options.model_W, options.model_n)
sq = generate_square(options.model_W, options.model_n)
joblib.dump((h2, v2, h3, v3, sq),
'%s%s_lc_feature_map.pkl' % (model_repo, options.model_name))
for id_term in term_list:
(dataset, rep, img) = build_dataset_image_offset_mp(
repository, Xc[:, term_to_i[id_term]], Yc[:, term_to_i[id_term]],
im_list, options.model_D_MAX, options.model_n_samples, h2, v2, h3,
v3, sq, options.model_njobs)
clf = VotingTreeRegressor(n_estimators=options.model_T,
n_jobs=options.model_njobs)
clf = clf.fit(dataset, rep)
joblib.dump(
clf, '%s%s_lc_regressor_%d.pkl' %
(model_repo, options.model_name, id_term))
xt = procrustes(Xc, Yc)
(mu, P) = apply_pca(xt, options.model_n_reduc)
joblib.dump((mu, P, np.mean(Xc, axis=0), np.mean(Yc, axis=0)),
'%s%s_lc_pca.pkl' % (model_repo, options.model_name))
F = open(
'%s%s_lc_parameters.conf' %
(options.model_save_to, options.model_name), 'wb')
F.write('cytomine_id_terms %s\n' % options.cytomine_id_terms)
F.write('model_njobs %d\n' % options.model_njobs)
F.write('model_D_MAX %d\n' % options.model_D_MAX)
F.write('model_n_samples %d\n' % options.model_n_samples)
F.write('model_W %d\n' % options.model_W)
F.write('model_n %d\n' % options.model_n)
F.write('model_T %d\n' % options.model_T)
F.write('model_step %d\n' % options.model_step)
F.write('model_n_reduc %d\n' % options.model_n_reduc)
F.write('model_R_MAX %d\n' % options.model_R_MAX)
F.write('model_R_MIN %d\n' % options.model_R_MIN)
F.write('model_alpha %f\n' % options.model_alpha)
F.close()
def main():
p = optparse.OptionParser(
description='Cytomine Landmark Detection : Model building',
prog='Cytomine Landmark Detector : Model builder',
version='0.1')
p.add_option(
'--cytomine_host',
type="string",
default='beta.cytomine.be',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='XXX',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='YYY',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option(
'--cytomine_training_images',
default="all",
type="string",
dest="cytomine_training_images",
help=
"identifiers of the images used to create the models. ids must be separated by commas (no spaces). If 'all' is mentioned instead, every image with manual annotation will be used."
)
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option(
'--image_type',
type='string',
default='jpg',
dest='image_type',
help="The type of the images that will be used (jpg, bmp, png,...)")
p.add_option('--model_njobs',
type='int',
default=1,
dest='model_njobs',
help="The number of processors used for model building")
p.add_option(
'--cytomine_id_terms',
type='string',
default=1,
dest='cytomine_id_terms',
help=
"The identifiers of the terms to create detection models for. Terms must be separated by commas (no spaces). If 'all' is mentioned instead, every terms will be detected."
)
p.add_option('--model_NT_P1',
type='int',
default=6,
dest='model_NT_P1',
help="Number of trees for phase 1.")
p.add_option('--model_F_P1',
type='int',
default=200,
dest='model_F_P1',
help="Number of features for phase 1.")
p.add_option('--model_R_P1',
type='int',
default=3,
dest='model_R_P1',
help="Radius in which phase 1 samples are extracted.")
p.add_option('--model_sigma',
type='int',
default=20,
dest='model_sigma',
help="Standard deviation for the gaussian.")
p.add_option('--model_delta',
type='float',
default=3,
dest='model_delta',
help="Resizing factor.")
p.add_option('--model_P',
type='float',
default=3,
dest='model_P',
help="Proportion of non-landmarks.")
p.add_option('--model_R_P2',
type='int',
default=3,
dest='model_R_P2',
help="Radius in which phase 2 samples are extracted.")
p.add_option('--model_ns_P2',
type='int',
default=3,
dest='model_ns_P2',
help="Number of samples for phase 2.")
p.add_option('--model_NT_P2',
type='int',
default=3,
dest='model_NT_P2',
help="Number of trees for phase 2.")
p.add_option('--model_F_P2',
type='int',
default=3,
dest='model_F_P2',
help="Number of features for phase 2.")
p.add_option('--model_filter_size',
type='int',
default=3,
dest='model_filter_size',
help="Size of the filter for phase 2.")
p.add_option('--model_beta',
type='float',
default=3,
dest='model_beta',
help="Beta for phase 2.")
p.add_option('--model_n_iterations',
type='int',
default=3,
dest='model_n_iterations',
help="Number of iterations for phase 2.")
p.add_option('--model_ncandidates',
type='int',
default=3,
dest='model_ncandidates',
help="Number of candidates for phase 3.")
p.add_option('--model_sde',
type='float',
default=10.,
dest='model_sde',
help="Standard deviation for gaussian phase 3.")
p.add_option('--model_T',
type='int',
default=3,
dest='model_T',
help="Number of edges for phase 3.")
p.add_option('--model_save_to',
type='string',
default='/tmp/',
dest='model_save_to',
help="Destination for model storage")
p.add_option('--model_name',
type='string',
dest='model_name',
help="Name of the model (used for saving)")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
opt_parser, arguments = p.parse_args(args=sys.argv)
opt_parser.cytomine_working_path = opt_parser.cytomine_working_path.rstrip(
'/') + '/'
cytomine_connection = cytomine.Cytomine(
opt_parser.cytomine_host,
opt_parser.cytomine_public_key,
opt_parser.cytomine_private_key,
base_path=opt_parser.cytomine_base_path,
working_path=opt_parser.cytomine_working_path,
verbose=str2bool(opt_parser.verbose))
current_user = cytomine_connection.get_current_user()
if not current_user.algo:
user_job = cytomine_connection.add_user_job(
opt_parser.cytomine_id_software, opt_parser.cytomine_id_project)
cytomine_connection.set_credentials(str(user_job.publicKey),
str(user_job.privateKey))
else:
user_job = current_user
run_by_user_job = True
job = cytomine_connection.get_job(user_job.job)
cytomine_connection.update_job_status(job,
status=job.RUNNING,
progress=0,
status_comment="Bulding model...")
job_parameters = {}
job_parameters['cytomine_id_terms'] = opt_parser.cytomine_id_terms
job_parameters['model_njobs'] = opt_parser.model_njobs
job_parameters['model_NT_P1'] = opt_parser.model_NT_P1
job_parameters['model_F_P1'] = opt_parser.model_F_P1
job_parameters['model_R_P1'] = opt_parser.model_R_P1
job_parameters['model_sigma'] = opt_parser.model_sigma
job_parameters['model_delta'] = opt_parser.model_delta
job_parameters['model_P'] = opt_parser.model_P
job_parameters['model_R_P2'] = opt_parser.model_R_P2
job_parameters['model_ns_P2'] = opt_parser.model_ns_P2
job_parameters['model_NT_P2'] = opt_parser.model_NT_P2
job_parameters['model_F_P2'] = opt_parser.model_F_P2
job_parameters['model_filter_size'] = opt_parser.model_filter_size
job_parameters['model_beta'] = opt_parser.model_beta
job_parameters['model_n_iterations'] = opt_parser.model_n_iterations
job_parameters['model_ncandidates'] = opt_parser.model_ncandidates
job_parameters['model_sde'] = opt_parser.model_sde
job_parameters['model_T'] = opt_parser.model_T
model_repo = opt_parser.model_save_to
if not os.path.isdir(model_repo):
os.mkdir(model_repo)
if not run_by_user_job:
cytomine_connection.add_job_parameters(
user_job.job,
cytomine_connection.get_software(opt_parser.cytomine_id_software),
job_parameters)
download_images(cytomine_connection, opt_parser.cytomine_id_project)
download_annotations(cytomine_connection, opt_parser.cytomine_id_project,
opt_parser.cytomine_working_path)
repository = opt_parser.cytomine_working_path + str(
opt_parser.cytomine_id_project) + '/'
(xc, yc, xr, yr, ims, term_to_i,
i_to_term) = getallcoords(repository.rstrip('/') + '/txt/')
(nims, nldms) = xc.shape
if opt_parser.cytomine_id_terms != 'all':
term_list = np.sort(
[int(term) for term in opt_parser.cytomine_id_terms.split(',')])
Xc = np.zeros((nims, len(term_list)))
Yc = np.zeros(Xc.shape)
i = 0
for id_term in term_list:
Xc[:, i] = xc[:, term_to_i[id_term]]
Yc[:, i] = yc[:, term_to_i[id_term]]
i += 1
else:
term_list = np.sort(term_to_i.keys())
Xc = xc
Yc = yc
if opt_parser.cytomine_training_images == 'all':
tr_im = ims
else:
tr_im = [
int(p) for p in opt_parser.cytomine_training_images.split(',')
]
(dataset, rep, img,
feature_offsets_1) = build_phase_1_model(repository,
tr_image=tr_im,
image_ids=ims,
n_jobs=opt_parser.model_njobs,
NT=opt_parser.model_NT_P1,
F=opt_parser.model_F_P1,
R=opt_parser.model_R_P1,
sigma=opt_parser.model_sigma,
delta=opt_parser.model_delta,
P=opt_parser.model_P,
X=Xc,
Y=Yc)
clf = SeparateTrees(n_estimators=opt_parser.model_NT_P1,
n_jobs=opt_parser.model_njobs)
clf = clf.fit(dataset, rep)
joblib.dump(
clf,
'%s%s_classifier_phase1.pkl' % (model_repo, opt_parser.model_name))
joblib.dump(
feature_offsets_1,
'%s%s_offsets_phase1.pkl' % (model_repo, opt_parser.model_name))
for id_term in term_list:
(dataset, rep, number, feature_offsets_2) = build_phase_2_model(
repository,
tr_image=tr_im,
image_ids=ims,
n_jobs=opt_parser.model_njobs,
IP=id_term,
NT=opt_parser.model_NT_P2,
F=opt_parser.model_F_P2,
R=opt_parser.model_R_P2,
N=opt_parser.model_ns_P2,
sigma=opt_parser.model_sigma,
delta=opt_parser.model_delta)
reg = SeparateTreesRegressor(n_estimators=opt_parser.model_NT_P2,
n_jobs=opt_parser.model_njobs)
reg.fit(dataset, rep)
joblib.dump(
reg, '%s%s_dmbl_regressor_phase2_%d.pkl' %
(model_repo, opt_parser.model_name, id_term))
joblib.dump(
feature_offsets_2, '%s%s_dmbl_offsets_phase2_%d.pkl' %
(model_repo, opt_parser.model_name, id_term))
(nims, nldms) = xc.shape
X = np.zeros((len(tr_im), nldms))
Y = np.zeros(X.shape)
j = 0
for i in range(nims):
if ims[i] in tr_im:
X[j, :] = xc[i, :]
Y[j, :] = yc[i, :]
j += 1
edges = build_edgematrix_phase_3(X, Y, opt_parser.model_sde,
opt_parser.model_delta,
opt_parser.model_T)
joblib.dump(
edges, '%s%s_edgematrix_phase3.pkl' %
(opt_parser.model_save_to, opt_parser.model_name))
F = open(
'%s%s_dmbl_parameters.conf' %
(opt_parser.model_save_to, opt_parser.model_name), 'wb')
F.write('cytomine_id_terms %s\n' % opt_parser.cytomine_id_terms)
F.write('model_njobs %d\n' % opt_parser.model_njobs)
F.write('model_NT_P1 %d\n' % opt_parser.model_NT_P1)
F.write('model_F_P1 %d\n' % opt_parser.model_F_P1)
F.write('model_R_P1 %d\n' % opt_parser.model_R_P1)
F.write('model_sigma %f\n' % opt_parser.model_sigma)
F.write('model_delta %f\n' % opt_parser.model_delta)
F.write('model_P %f\n' % opt_parser.model_P)
F.write('model_R_P2 %d\n' % opt_parser.model_R_P2)
F.write('model_ns_P2 %d\n' % opt_parser.model_ns_P2)
F.write('model_NT_P2 %d\n' % opt_parser.model_NT_P2)
F.write('model_F_P2 %d\n' % opt_parser.model_F_P2)
F.write('model_filter_size %d\n' % opt_parser.model_filter_size)
F.write('model_beta %f\n' % opt_parser.model_beta)
F.write('model_n_iterations %d\n' % opt_parser.model_n_iterations)
F.write("model_ncandidates %d\n" % opt_parser.model_ncandidates)
F.write('model_sde %f\n' % opt_parser.model_sde)
F.write('model_T %d' % opt_parser.model_T)
F.close()
def main(argv):
# Define command line options
p = optparse.OptionParser(
description='Pyxit/Cytomine Classification Model Prediction',
prog='PyXit Classification Model Prediction (PYthon piXiT)')
p.add_option(
"--cytomine_host",
type="string",
default='',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--pyxit_save_to',
type='string',
dest='pyxit_save_to',
help="pyxit model file") # future: get it from server db
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-i',
'--cytomine_id_image',
type="int",
dest="cytomine_id_image",
help="The Cytomine image identifier")
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('--cytomine_dump_type',
type='int',
dest='cytomine_dump_type',
help="dump type of annotations (with/out alpha channel)")
p.add_option(
'--cytomine_id_userjob',
type="int",
dest="cytomine_id_userjob",
help=
"The Cytomine user (job) id of annotations to classify with the model")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=argv)
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_software'] = options.cytomine_id_software
# to define which annotations we will classify
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_image'] = options.cytomine_id_image
parameters['cytomine_id_userjob'] = options.cytomine_id_userjob
# to define which resolution and image type we will classify
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['cytomine_dump_type'] = options.cytomine_dump_type
parameters['pyxit_save_to'] = options.pyxit_save_to
print "[pyxit.main] Options = ", options
# Create JOB/USER/JOB
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
# Create a new userjob if connected as human user
current_user = conn.get_current_user()
run_by_user_job = False
if current_user.algo == False:
print "adduserJob..."
user_job = conn.add_user_job(parameters['cytomine_id_software'],
parameters['cytomine_id_project'])
print "set_credentials..."
conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey))
print "done"
else:
user_job = current_user
print "Already running as userjob"
run_by_user_job = True
job = conn.get_job(user_job.job)
print "Fetching data..."
job = conn.update_job_status(
job, status_comment="Publish software parameters values")
if run_by_user_job == False:
job_parameters_values = conn.add_job_parameters(
user_job.job,
conn.get_software(parameters['cytomine_id_software']), parameters)
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Run...",
progress=0)
# Image dump type (for classification use 1)
if parameters['cytomine_dump_type'] == 1:
annotation_get_func = Annotation.get_annotation_crop_url
elif parameters['cytomine_dump_type'] == 2:
annotation_get_func = Annotation.get_annotation_alpha_crop_url
else:
print "default annotation type crop"
annotation_get_func = Annotation.get_annotation_crop_url
# Get description of annotations to predict (e.g. geometries created by a another object finder (e.g. threshold) job)
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Run (getting annotations)...",
progress=25)
candidate_annotations = conn.get_annotations(
id_user=parameters['cytomine_id_userjob'],
id_image=parameters['cytomine_id_image'],
id_project=parameters['cytomine_id_project'],
showWKT=True,
showMeta=True)
print "Number of annotations to predict: %d" % len(
candidate_annotations.data())
time.sleep(2)
# Create temporary dir to download annotation crops
folder_name = "%s/annotations/project-%d/crops-candidates-%d-%d/zoom-%d/" % (
parameters["cytomine_working_path"],
parameters["cytomine_id_project"],
parameters["cytomine_id_image"],
job.userJob, # current job
parameters['cytomine_zoom_level'])
if not os.path.exists(folder_name):
os.makedirs(folder_name)
# Load Classifier model
classifier = open(parameters['pyxit_save_to'], "r")
classes = pickle.load(classifier)
pyxit = pickle.load(classifier)
print "Model: %s" % pyxit
time.sleep(2)
print "Dumping annotation cropped images to classify to %s" % folder_name
annotation_mapping = {}
for i, annotation in enumerate(candidate_annotations.data()):
url = annotation_get_func(
annotation, desired_zoom=parameters['cytomine_zoom_level'])
filename = folder_name + str(annotation.id) + ".png"
annotation_mapping[annotation.id] = filename
conn.fetch_url_into_file(url, filename, False, True)
np_image = cv2.imread(filename, -1)
if parameters['cytomine_dump_type'] == 2 and np_image is not None:
alpha = np.array(np_image[:, :, 3])
image = np.array(np_image[:, :, 0:3])
image[alpha == 0] = (255, 255, 255) # to replace alpha by white
cv2.imwrite(filename, image)
print "Building subwindows from ", os.path.dirname(
os.path.dirname(folder_name))
# Extract subwindows from all candidates annotations
X, y = pyxitstandalone.build_from_dir(
os.path.dirname(os.path.dirname(folder_name)))
# Apply pyxit classifier model to X (parameters are already reused from model pkl file)
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Run (applying Classifier)...",
progress=50)
y_proba = pyxit.predict_proba(X)
y_predict = classes.take(np.argmax(y_proba, axis=1), axis=0)
y_rate = np.max(y_proba, axis=1)
# Creating new annotations on Cytomine with predicted terms by current classifier model
for k, annotation in enumerate(candidate_annotations.data()):
filename = annotation_mapping[annotation.id]
j = np.where(X == filename)[0][0]
print "Annotation filename %s id: %d class: %d proba: %d" % (
filename, annotation.id, int(y_predict[j]), y_rate[j])
new_annotation = conn.add_annotation(annotation.location,
parameters["cytomine_id_image"])
conn.add_annotation_term(
new_annotation.id,
int(y_predict[j]),
int(y_predict[j]),
y_rate[j],
annotation_term_model=models.AlgoAnnotationTerm)
job = conn.update_job_status(job,
status=job.TERMINATED,
progress=100,
status_comment="Finish Job..")
print "END."
if __name__ == "__main__":
import cytomine
# Connect to cytomine, edit connection values
cytomine_host = "http://localhost-core:8080"
Pk = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
Prk = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
id_project = 19941904
# Connection to Cytomine Core
conn = cytomine.Cytomine(cytomine_host,
Pk,
Prk,
base_path='/api/',
working_path=os.path.join(tempfile.gettempdir(),
"cytomine"),
verbose=True)
#add software with execute command
software = conn.add_software(
"Simple_Elastix", "createRabbitJobWithArgsService", "download",
"/root/miniconda2/envs/cytomine/bin/python software_routeur/algo/simple_elastix/get_and_move.py "
+ "--cytomine_host $cytomine_host " +
"--cytomine_public_key $cytomine_public_key " +
"--cytomine_private_key $cytomine_private_key " +
"--cytomine_id_software $cytomine_id_software " +
"--cytomine_working_path software_router/algo/simple_elastix " +
"--cytomine_id_project $cytomine_id_project " +
"--fix_image_id fix_image " + "--mov_image_id $id_mov_image " +
def main(argv):
parser = optparse.OptionParser(description='Cytomine Datamining',
prog='cytomining',
version='cytomining 0.1')
parser.add_option('--cytomine_host',
dest='cytomine_host',
help='cytomine_host')
parser.add_option('--cytomine_public_key',
dest='cytomine_public_key',
help='cytomine_public_key')
parser.add_option('--cytomine_private_key',
dest='cytomine_private_key',
help='cytomine_private_key')
parser.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
parser.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
parser.add_option('--cytomine_base_path',
dest='cytomine_base_path',
help='cytomine base path')
parser.add_option('--cytomine_working_path',
dest='cytomine_working_path',
help='cytomine_working_path base path')
options, arguments = parser.parse_args(args=argv)
# copy options
parameters = {}
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
print "Connection to Cytomine server"
cytomine_connection = cytomine.Cytomine(
parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'])
# Create a new userjob if connected as human user
current_user = cytomine_connection.get_current_user()
if current_user.algo == False:
print "adduserJob..."
user_job = cytomine_connection.add_user_job(
parameters['cytomine_id_software'],
parameters['cytomine_id_project'])
print "set_credentials..."
cytomine_connection.set_credentials(str(user_job.publicKey),
str(user_job.privateKey))
print "done"
else:
user_job = current_user
print "Already running as userjob"
job = cytomine_connection.get_job(user_job.job)
job = cytomine_connection.update_job_status(job,
status=job.RUNNING,
status_comment="Fetching data",
progress=0)
images = cytomine_connection.get_project_image_instances(
parameters['cytomine_id_project'])
images = images.data()
xpos = {}
ypos = {}
terms = {}
progress = 0
delta = 80 / len(images)
for image in images:
progress += delta
job = cytomine_connection.update_job_status(
job,
status=job.RUNNING,
status_comment="Fetching data",
progress=progress)
annotations = cytomine_connection.get_annotations(
id_project=parameters['cytomine_id_project'],
showWKT=True,
id_image=image.id,
reviewed_only=True)
ann_data = annotations.data()
for ann in ann_data:
if (len(ann.term) > 0):
l = ann.location
if (l.rfind('POINT') == -1):
pol = shapely.wkt.loads(l)
poi = pol.centroid
else:
poi = shapely.wkt.loads(l)
(cx, cy) = poi.xy
xpos[(ann.term[0], image.id)] = int(cx[0])
ypos[(ann.term[0], image.id)] = image.height - int(cy[0])
term = cytomine_connection.get_term(ann.term[0])
terms[term.id] = term.name
key_t = terms.keys()
job = cytomine_connection.update_job_status(job,
status=job.RUNNING,
status_comment="Write in file",
progress=90)
if not os.path.exists(parameters['cytomine_working_path']):
os.makedirs(parameters['cytomine_working_path'])
csv = open(
'%s%s.csv' % (parameters['cytomine_working_path'],
str(parameters['cytomine_id_project'])), 'w')
csv.write('ID_IMAGE;')
for i in range(len(key_t)):
csv.write('%s_x;%s_y;' % (str(terms[key_t[i]]), str(terms[key_t[i]])))
csv.write('\n')
for image in images:
csv.write('%s;' % str(image.id))
for i in range(len(key_t)):
if ((key_t[i], image.id) in xpos):
csv.write(
'%3.3f;%3.3f;' %
(xpos[(key_t[i], image.id)], ypos[(key_t[i], image.id)]))
else:
csv.write('-1;-1;')
csv.write('\n')
csv.close()
job_data = cytomine_connection.add_job_data(
job,
key=job.id,
filename='%s%s.csv' % (parameters['cytomine_working_path'],
str(parameters['cytomine_id_project'])))
job = cytomine_connection.update_job_status(
job,
status=job.TERMINATED,
status_comment="File available",
progress=100)
print "File available"
def main(argv):
# Define command line options
p = optparse.OptionParser(
description='Pyxit/Cytomine Classification Model Builder',
prog='PyXit Classification Model Builder (PYthon piXiT)')
p.add_option(
"--cytomine_host",
type="string",
default='',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('--cytomine_dump_type',
type='int',
dest='cytomine_dump_type',
help="annotation type (1=crop, 2=alphamask)")
p.add_option('--cytomine_annotation_projects',
type="string",
dest="cytomine_annotation_projects",
help="Projects from which annotations are extracted")
p.add_option(
'--cytomine_predict_terms',
type='string',
default='0',
dest='cytomine_predict_terms',
help="term ids of predicted terms (=positive class in binary mode)")
p.add_option('--cytomine_excluded_terms',
type='string',
default='0',
dest='cytomine_excluded_terms',
help="term ids of excluded terms")
#p.add_option('--cytomine_reviewed', default=False, action="store_true", dest="cytomine_reviewed", help="Get reviewed annotations only")
p.add_option('--cytomine_reviewed',
type='string',
default="False",
dest="cytomine_reviewed",
help="Get reviewed annotations only")
p.add_option('--pyxit_target_width',
type='int',
dest='pyxit_target_width',
help="pyxit subwindows width")
p.add_option('--pyxit_target_height',
type='int',
dest='pyxit_target_height',
help="pyxit subwindows height")
p.add_option('--pyxit_save_to',
type='string',
dest='pyxit_save_to',
help="pyxit model directory") #future: get it from server db
p.add_option(
'--pyxit_colorspace',
type='int',
dest='pyxit_colorspace',
help="pyxit colorspace encoding") #future: get it from server db
p.add_option(
'--pyxit_n_jobs',
type='int',
dest='pyxit_n_jobs',
help="pyxit number of jobs for trees") #future: get it from server db
p.add_option('--pyxit_n_subwindows',
default=10,
type="int",
dest="pyxit_n_subwindows",
help="number of subwindows")
p.add_option('--pyxit_min_size',
default=0.5,
type="float",
dest="pyxit_min_size",
help="min size")
p.add_option('--pyxit_max_size',
default=1.0,
type="float",
dest="pyxit_max_size",
help="max size")
p.add_option('--pyxit_interpolation',
default=2,
type="int",
dest="pyxit_interpolation",
help="interpolation method 1,2,3,4")
#p.add_option('--pyxit_transpose', default=False, action="store_true", dest="pyxit_transpose", help="transpose subwindows")
p.add_option('--pyxit_transpose',
type="string",
default="False",
dest="pyxit_transpose",
help="transpose subwindows")
#p.add_option('--pyxit_fixed_size', default=False, action="store_true", dest="pyxit_fixed_size", help="extract fixed size subwindows")
p.add_option('--pyxit_fixed_size',
type="string",
default="False",
dest="pyxit_fixed_size",
help="extract fixed size subwindows")
p.add_option('--forest_n_estimators',
default=10,
type="int",
dest="forest_n_estimators",
help="number of base estimators (T)")
p.add_option('--forest_max_features',
default=1,
type="int",
dest="forest_max_features",
help="max features at test node (k)")
p.add_option('--forest_min_samples_split',
default=1,
type="int",
dest="forest_min_samples_split",
help="minimum node sample size (nmin)")
p.add_option('--forest_shared_mem',
default=False,
action="store_true",
dest="forest_shared_mem",
help="shared mem")
p.add_option(
'--svm',
default=0,
dest="svm",
help=
"final svm classifier: 0=nosvm, 1=libsvm, 2=liblinear, 3=lr-l1, 4=lr-l2",
type="int")
p.add_option('--svm_c',
default=1.0,
type="float",
dest="svm_c",
help="svm C")
#p.add_option('--verbose', action="store_true", default=True, dest="verbose", help="Turn on verbose mode")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=argv)
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_annotation_projects'] = map(
int, options.cytomine_annotation_projects.split(','))
parameters['cytomine_predict_terms'] = map(
int, options.cytomine_predict_terms.split(','))
parameters['cytomine_excluded_terms'] = map(
int, options.cytomine_excluded_terms.split(','))
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['cytomine_dump_type'] = options.cytomine_dump_type
parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed)
pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width
pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height
pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size
pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size
pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace
pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation
pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators
pyxit_parameters['forest_max_features'] = options.forest_max_features
pyxit_parameters[
'forest_min_samples_split'] = options.forest_min_samples_split
pyxit_parameters['forest_shared_mem'] = options.forest_min_samples_split
pyxit_parameters['svm'] = options.svm
pyxit_parameters['svm_c'] = options.svm_c
pyxit_parameters['pyxit_save_to'] = options.pyxit_save_to
pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs
# Check for errors in the options
if options.verbose:
print "[pyxit.main] Options = ", options
# Create JOB/USER/JOB
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
#Create a new userjob if connected as human user
current_user = conn.get_current_user()
run_by_user_job = False
if current_user.algo == False:
print "adduserJob..."
user_job = conn.add_user_job(parameters['cytomine_id_software'],
parameters['cytomine_id_project'])
print "set_credentials..."
conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey))
print "done"
else:
user_job = current_user
print "Already running as userjob"
run_by_user_job = True
job = conn.get_job(user_job.job)
print "Fetching data..."
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Run...",
progress=0)
#Get annotation descriptions (JSON) from project(s)
annotations = None
for prj in parameters['cytomine_annotation_projects']:
if parameters["cytomine_reviewed"]:
print "Retrieving reviewed annotations..."
annotations_prj = conn.get_annotations(
id_project=prj, reviewed_only=parameters["cytomine_reviewed"])
print "Reviewed annotations: %d" % len(annotations_prj.data())
else:
print "Retrieving (unreviewed) annotations..."
annotations_prj = conn.get_annotations(id_project=prj)
print "(Unreviewed) annotations: %d" % len(annotations_prj.data())
if not annotations:
annotations = annotations_prj
else:
annotations.data().extend(annotations_prj.data())
print "Nb annotations so far... = %d" % len(annotations.data())
print "Total annotations projects %s = %d" % (
parameters['cytomine_annotation_projects'], len(annotations.data()))
time.sleep(2)
#Set output dir parameters
pyxit_parameters['dir_ls'] = os.path.join(
parameters["cytomine_working_path"],
str(parameters['cytomine_annotation_projects']).replace(
',', '-').replace('[', '').replace(']', '').replace(' ', ''),
"zoom_level", str(parameters['cytomine_zoom_level']), "dump_type",
str(parameters['cytomine_dump_type']))
if not os.path.exists(pyxit_parameters['dir_ls']):
print "Creating annotation directory: %s" % pyxit_parameters['dir_ls']
os.makedirs(pyxit_parameters['dir_ls'])
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Dump annotations...",
progress=50)
time.sleep(2)
#Image dump type (for classification use 1)
if (parameters['cytomine_dump_type'] == 1):
annotation_get_func = Annotation.get_annotation_crop_url
elif (parameters['cytomine_dump_type'] == 2):
annotation_get_func = Annotation.get_annotation_alpha_crop_url
else:
print "default annotation type crop"
annotation_get_func = Annotation.get_annotation_crop_url
#Dump annotation images locally
annotations = conn.dump_annotations(
annotations=annotations,
get_image_url_func=annotation_get_func,
dest_path=pyxit_parameters['dir_ls'],
desired_zoom=parameters['cytomine_zoom_level'],
excluded_terms=parameters['cytomine_excluded_terms'])
print "END dump annotations"
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Build model...",
progress=75)
#Build pyxit classification model (and saves it)
print "Upload software parameters values to Cytomine-Core..."
if run_by_user_job == False:
parameters_values = conn.add_job_parameters(
user_job.job,
conn.get_software(parameters['cytomine_id_software']),
pyxit_parameters)
argv = parameters_values_to_argv(pyxit_parameters, parameters_values)
else:
argv = []
for key in pyxit_parameters:
value = pyxit_parameters[key]
if type(value) is bool or value == 'True':
if bool(value):
argv.append("--%s" % key)
elif not value == 'False':
argv.append("--%s" % key)
argv.append("%s" % value)
print "argv :"
print argv
print "Run PyXiT..."
d = os.path.dirname(pyxit_parameters['pyxit_save_to'])
if not os.path.exists(d):
os.makedirs(d)
predict = pyxitstandalone.main(argv)
print "-------------------------------------------------------"
print "Pyxit Classification Model saved locally: %s " % pyxit_parameters[
'pyxit_save_to']
print "-------------------------------------------------------"
job = conn.update_job_status(job,
status=job.TERMINATED,
status_comment="Finish",
progress=100)
def main(argv):
current_path = os.getcwd() + '/' + os.path.dirname(__file__)
# Define command line options
p = optparse.OptionParser(description='Cytomine Segmentation prediction',
prog='Cytomine segmentation prediction',
version='0.1')
p.add_option(
'--cytomine_host',
type="string",
default='beta.cytomine.be',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-i',
'--cytomine_id_image',
type='int',
dest='cytomine_id_image',
help="image id from cytomine",
metavar='IMAGE')
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('-j',
'--nb_jobs',
type='int',
dest='nb_jobs',
help="number of parallel jobs")
p.add_option(
'--cytomine_predict_terms',
type='str',
dest='cytomine_predict_terms',
help=
"term id of all positive terms. The first term is the output predicted annotation term"
)
p.add_option('--cytomine_excluded_terms',
type='string',
dest='cytomine_excluded_terms',
help="term id of excluded terms)")
p.add_option('--pyxit_target_width',
type='int',
dest='pyxit_target_width',
help="pyxit subwindows width")
p.add_option('--pyxit_target_height',
type='int',
dest='pyxit_target_height',
help="pyxit subwindows height")
p.add_option('--pyxit_colorspace',
type='int',
dest='pyxit_colorspace',
help="pyxit colorspace encoding")
p.add_option('--pyxit_nb_jobs',
type='int',
dest='pyxit_nb_jobs',
help="pyxit number of jobs for trees")
p.add_option('--pyxit_fixed_size',
type='string',
default="0",
dest="pyxit_fixed_size",
help="extract fixed size subwindows")
p.add_option('--pyxit_transpose',
type='string',
default="0",
dest="pyxit_transpose",
help="transpose subwindows")
p.add_option('--pyxit_n_subwindows',
type='int',
default="10",
dest="pyxit_n_subwindows",
help="number of subwindows")
p.add_option('--pyxit_interpolation',
default=2,
type="int",
dest="pyxit_interpolation",
help="interpolation method 1,2,3,4")
p.add_option('--pyxit_min_size',
default=0.5,
type="float",
dest="pyxit_min_size",
help="min size")
p.add_option('--pyxit_max_size',
default=1.0,
type="float",
dest="pyxit_max_size",
help="max size")
p.add_option('--cytomine_reviewed',
type='string',
default="False",
dest="cytomine_reviewed",
help="Get reviewed annotations only")
p.add_option('--cytomine_dump_annotations',
type='string',
default="0",
dest="cytomine_dump_annotations",
help="Dump training annotations or not")
p.add_option('--cytomine_dump_annotation_stats',
type='string',
default="0",
dest="cytomine_dump_annotation_stats",
help="Calculate stats on dumped annotations or not")
p.add_option('--build_model',
type="string",
default="0",
dest="build_model",
help="Turn on (1) or off (0) model building")
p.add_option('--cytomine_annotation_projects',
type="string",
dest="cytomine_annotation_projects",
help="Projects from which annotations are extracted")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
p.add_option('--keras_save_to',
type='string',
default="",
dest='keras_save_to',
help="keras model weight file")
p.add_option('--keras_batch_size',
type="int",
dest="keras_batch_size",
help="Training batch size")
p.add_option('--keras_n_epochs',
type="int",
dest="keras_n_epochs",
help="Number of epochs")
p.add_option('--keras_shuffle',
type="string",
dest="keras_shuffle",
help="Turn on (1) or off (0) batch shuffle")
p.add_option('--keras_validation_split',
type="float",
dest="keras_validation_split",
help="Batch validation split")
options, arguments = p.parse_args(args=argv)
parameters = {}
parameters['keras_save_to'] = options.keras_save_to
parameters['keras_batch_size'] = options.keras_batch_size
parameters['keras_n_epochs'] = options.keras_n_epochs
parameters['keras_shuffle'] = options.keras_shuffle
parameters['keras_validation_split'] = options.keras_validation_split
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_predict_terms'] = map(
int, options.cytomine_predict_terms.split(','))
parameters['cytomine_predicted_annotation_term'] = parameters[
'cytomine_predict_terms'][0]
parameters['cytomine_excluded_terms'] = map(
int, options.cytomine_excluded_terms.split(','))
parameters['pyxit_colorspace'] = options.pyxit_colorspace
parameters['pyxit_nb_jobs'] = options.pyxit_nb_jobs
parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs
parameters['cytomine_nb_jobs'] = options.pyxit_nb_jobs
parameters['cytomine_id_image'] = options.cytomine_id_image
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['nb_jobs'] = options.nb_jobs
parameters['pyxit_target_width'] = options.pyxit_target_width
parameters['pyxit_target_height'] = options.pyxit_target_height
parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
parameters['pyxit_interpolation'] = options.pyxit_interpolation
parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
parameters['pyxit_min_size'] = options.pyxit_min_size
parameters['pyxit_max_size'] = options.pyxit_max_size
parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
parameters['cytomine_annotation_projects'] = map(
int, options.cytomine_annotation_projects.split(','))
parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed)
parameters['cytomine_dump_annotation_stats'] = str2bool(
options.cytomine_dump_annotation_stats)
parameters['cytomine_dump_annotations'] = str2bool(
options.cytomine_dump_annotations)
parameters['build_model'] = str2bool(options.build_model)
parameters['dir_ls'] = os.path.join(
parameters["cytomine_working_path"],
str(parameters['cytomine_annotation_projects']).replace(
',', '-').replace('[', '').replace(']', '').replace(' ', ''),
"zoom_level", str(parameters['cytomine_zoom_level']))
pyxit_parameters = {}
pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width
pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height
pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size
pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size
pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace
pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation
pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
pyxit_parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs
if options.verbose:
print(parameters)
# Create Cytomine connection
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
# Dump annotations
if parameters['cytomine_dump_annotations']:
# Get annotation descriptions (JSON) from project(s)
annotations = None
for prj in parameters['cytomine_annotation_projects']:
if parameters["cytomine_reviewed"]:
annotations_prj = conn.get_annotations(
id_project=prj,
reviewed_only=parameters["cytomine_reviewed"])
else:
annotations_prj = conn.get_annotations(id_project=prj)
if not annotations:
annotations = annotations_prj
else:
annotations.data().extend(annotations_prj.data())
if prj == 21907448 or prj == 155194683:
annotations_prj = conn.get_annotations(id_project=prj,
id_term=91376951)
annotations.data().extend(annotations_prj.data())
print("Nb annotations so far... = %d" % len(annotations.data()))
print("Total annotations projects %s = %d" %
(parameters['cytomine_annotation_projects'],
len(annotations.data())))
# Set output dir parameters
if not os.path.exists(parameters['dir_ls']):
print("Creating annotation directory: %s" % parameters['dir_ls'])
os.makedirs(parameters['dir_ls'])
# Dump annotation images locally
print("Dump training annotation images in %s...", parameters['dir_ls'])
conn.dump_annotations(
annotations=annotations,
get_image_url_func=Annotation.get_annotation_alpha_crop_url,
dest_path=parameters['dir_ls'],
desired_zoom=parameters['cytomine_zoom_level'],
excluded_terms=parameters['cytomine_excluded_terms'])
# Put positive terms under the same term and same for negative terms
term_directories = os.listdir(parameters['dir_ls'])
pos_image_path = os.path.join(parameters['dir_ls'], "image", "1")
pos_mask_path = os.path.join(parameters['dir_ls'], "mask", "1")
neg_image_path = os.path.join(parameters['dir_ls'], "image", "0")
neg_mask_path = os.path.join(parameters['dir_ls'], "mask", "0")
if not os.path.exists(pos_image_path):
print("Creating positive annotation directory: %s" %
pos_image_path)
os.makedirs(pos_image_path)
if not os.path.exists(neg_image_path):
print("Creating negative annotation directory: %s" %
neg_image_path)
os.makedirs(neg_image_path)
if not os.path.exists(pos_mask_path):
print("Creating positive annotation directory: %s" % pos_mask_path)
os.makedirs(pos_mask_path)
if not os.path.exists(neg_mask_path):
print("Creating negative annotation directory: %s" % neg_mask_path)
os.makedirs(neg_mask_path)
for dir in term_directories:
dir_abs = os.path.join(parameters['dir_ls'], dir)
# Move files
print("Term directory: %s" % dir_abs)
if int(dir) in parameters['cytomine_predict_terms']:
print("Positive term")
for image_file in os.listdir(dir_abs):
print(image_file)
try:
im = Image.open(os.path.join(dir_abs, image_file))
except IOError:
"warning filename %s is not an image" % os.path.join(
dir_abs, image_file)
continue
rgb = im.tobytes("raw", "RGB")
a = im.tobytes("raw", "A")
im.close()
image = Image.frombytes("RGB", im.size, rgb)
mask = Image.frombytes("L", im.size, a)
image.save(os.path.join(pos_image_path, image_file), "PNG")
mask.save(os.path.join(pos_mask_path, image_file), "PNG")
else:
print("Negative term")
for image_file in os.listdir(dir_abs):
print(image_file)
try:
im = Image.open(os.path.join(dir_abs, image_file))
except IOError:
"warning filename %s is not an image" % os.path.join(
dir_abs, image_file)
continue
rgb = im.tobytes("raw", "RGB")
a = im.tobytes("raw", "A")
im.close()
image = Image.frombytes("RGB", im.size, rgb)
mask = Image.frombytes("L", im.size, a)
image.save(os.path.join(neg_image_path, image_file), "PNG")
mask.save(os.path.join(neg_mask_path, image_file), "PNG")
if parameters['cytomine_dump_annotation_stats']:
pos_path = os.path.join(parameters['dir_ls'], "image", "1")
neg_path = os.path.join(parameters['dir_ls'], "image", "0")
stats_dumped_annotations(pos_path, neg_path)
# if parameters['build_model'] :
# # Model name
# model_name = "nsubw{}_winsize{}x{}_minsize{}_maxsize{}_batchsize{}_epochs{}_shuffle{}_valsplit{}_colorspace{}"\
# .format(parameters['pyxit_n_subwindows'],
# parameters['pyxit_target_width'],
# parameters['pyxit_target_height'],
# parameters['pyxit_min_size'],
# parameters['pyxit_max_size'],
# parameters['keras_batch_size'],
# parameters['keras_n_epochs'],
# parameters['keras_shuffle'],
# parameters['keras_validation_split'],
# pyxit_parameters['pyxit_colorspace']).replace(".", "")
# print("Model_name :", model_name)
#
# pyxit = PyxitClassifier(None,
# n_subwindows = pyxit_parameters['pyxit_n_subwindows'],
# min_size = pyxit_parameters['pyxit_min_size'],
# max_size = pyxit_parameters['pyxit_max_size'],
# target_width = pyxit_parameters['pyxit_target_width'],
# target_height = pyxit_parameters['pyxit_target_height'],
# n_jobs = pyxit_parameters['pyxit_n_jobs'],
# interpolation = pyxit_parameters['pyxit_interpolation'],
# transpose = pyxit_parameters['pyxit_transpose'],
# colorspace = pyxit_parameters['pyxit_colorspace'],
# fixed_size = pyxit_parameters['pyxit_fixed_size'],
# random_state = None,
# verbose = 1,
# get_output = _get_output_from_mask,
# parallel_leaf_transform = False)
#
# # Build filenames and classes
# X, y = build_from_dir(parameters['dir_ls'])
#
# classes = np.unique(y)
# n_classes = len(classes)
# y_original = y
# y = np.searchsorted(classes, y)
# n_images = len(y)
# print("Number of images : ", n_images)
#
# # Extract subwindows
# _X, _y = pyxit.extract_subwindows(X, y)
# n_subw = len(_y)
# print("Number of subwindows : ", n_subw)
#
# # Reshape data structure
# _X = np.reshape(_X, (
# n_subw, pyxit_parameters['pyxit_target_width'], pyxit_parameters['pyxit_target_height'], n_channels))
# _y = np.reshape(_y, (n_subw, pyxit_parameters['pyxit_target_width'], pyxit_parameters['pyxit_target_height']))
#
# # Train FCN
# if not os.path.exists(parameters['keras_save_to']) :
# os.makedirs(parameters['keras_save_to'])
#
# model_weights_file_path = os.path.join(parameters['keras_save_to'], "weights_" + model_name + ".h5")
#
# mean, std = train(_X, _y,
# model_weights_file_path,
# imgs_width = pyxit_parameters['pyxit_target_width'],
# imgs_height = pyxit_parameters['pyxit_target_height'],
# batch_size = parameters['keras_batch_size'],
# epochs = parameters['keras_n_epochs'],
# shuffle = parameters['keras_shuffle'],
# validation_split = parameters['keras_validation_split'])
#
# # Save mean and std used to normalize training data
# mean_std_save_file_path = os.path.join(parameters['keras_save_to'], "meanstd_" + model_name + ".txt")
# mean_std_save_file = open(mean_std_save_file_path, 'w')
# mean_std_save_file.write(str(mean) + '\n')
# mean_std_save_file.write(str(std) + '\n')
if parameters['build_model']:
# Model name
model_name = "nsubw{}_winsize{}x{}_minsize{}_maxsize{}_batchsize{}_epochs{}_shuffle{}_valsplit{}_colorspace{}_zoom{}_until4x4_IDG"\
.format(parameters['pyxit_n_subwindows'],
parameters['pyxit_target_width'],
parameters['pyxit_target_height'],
parameters['pyxit_min_size'],
parameters['pyxit_max_size'],
parameters['keras_batch_size'],
parameters['keras_n_epochs'],
parameters['keras_shuffle'],
parameters['keras_validation_split'],
pyxit_parameters['pyxit_colorspace'],
parameters['cytomine_zoom_level']).replace(".", "")
print("Model_name :", model_name)
pyxit = PyxitClassifier(
None,
n_subwindows=1,
min_size=1,
max_size=1,
target_width=pyxit_parameters['pyxit_target_width'],
target_height=pyxit_parameters['pyxit_target_height'],
n_jobs=pyxit_parameters['pyxit_n_jobs'],
interpolation=pyxit_parameters['pyxit_interpolation'],
transpose=pyxit_parameters['pyxit_transpose'],
colorspace=pyxit_parameters['pyxit_colorspace'],
fixed_size=pyxit_parameters['pyxit_fixed_size'],
random_state=None,
verbose=1,
get_output=_get_output_from_mask,
parallel_leaf_transform=False)
# pyxit = PyxitClassifier(None,
# n_subwindows=pyxit_parameters['pyxit_n_subwindows'],
# min_size=pyxit_parameters['pyxit_min_size'],
# max_size=pyxit_parameters['pyxit_max_size'],
# target_width=pyxit_parameters['pyxit_target_width'],
# target_height=pyxit_parameters['pyxit_target_height'],
# n_jobs=pyxit_parameters['pyxit_n_jobs'],
# interpolation=pyxit_parameters['pyxit_interpolation'],
# transpose=pyxit_parameters['pyxit_transpose'],
# colorspace=pyxit_parameters['pyxit_colorspace'],
# fixed_size=pyxit_parameters['pyxit_fixed_size'],
# random_state=None,
# verbose=1,
# get_output = _get_output_from_mask,
# parallel_leaf_transform=False)
# Build filenames and classes
X, y = build_from_dir(parameters['dir_ls'])
classes = np.unique(y)
n_classes = len(classes)
y_original = y
y = np.searchsorted(classes, y)
n_images = len(y)
print("Number of images : ", n_images)
print("Start extraction of subwindows...")
# Extract subwindows
_X, _y = pyxit.extract_subwindows(X, y)
print("Over")
n_subw = len(_y)
print("Number of subwindows : ", n_subw)
# Reshape data structure
_X = np.reshape(_X,
(n_subw, pyxit_parameters['pyxit_target_width'],
pyxit_parameters['pyxit_target_height'], n_channels))
_y = np.reshape(_y, (n_subw, pyxit_parameters['pyxit_target_width'],
pyxit_parameters['pyxit_target_height'], 1))
print(type(_X))
print(type(_y))
# ImageDataGenerator : two instances with the same arguments
print("Init data gen")
data_gen_args = dict(rotation_range=180.,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2,
rescale=1 / 255,
horizontal_flip=True,
vertical_flip=True)
# featurewise_center = True,
# featurewise_std_normalization = True)
image_datagen = ImageDataGenerator(**data_gen_args)
mask_datagen = ImageDataGenerator(**data_gen_args)
# Provide the same seed and keyword arguments to the fit and flow methods
seed = 1
print("Fit image data generator (image)...")
image_datagen.fit(_X[0:100], augment=True, seed=seed)
print("Fit image data generator (mask)...")
mask_datagen.fit(_y[0:100], augment=True, seed=seed)
print('Flow on images...')
# image_generator = image_datagen.flow(_X, labels, seed = seed, shuffle = False)
image_generator = image_datagen.flow_from_directory(
os.path.join(parameters['dir_ls'], "image"),
class_mode=None,
target_size=(128, 128),
seed=seed)
print('Flow on masks...')
# mask_generator = mask_datagen.flow(_y, labels, seed = seed, shuffle = False)
mask_generator = mask_datagen.flow_from_directory(
os.path.join(parameters['dir_ls'], "mask"),
class_mode=None,
target_size=(128, 128),
seed=seed)
# combine generators into one which yields image and masks
train_generator = combine_generator(image_generator, mask_generator)
# Creating and compiling model
if not os.path.exists(parameters['keras_save_to']):
os.makedirs(parameters['keras_save_to'])
model_weights_filename = os.path.join(parameters['keras_save_to'],
"weights_" + model_name + ".h5")
print('Fitting model...')
model = get_unet(128, 128)
model_checkpoint = ModelCheckpoint(model_weights_filename,
monitor='val_loss',
save_best_only=True)
# Train FCN
model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=50,
callbacks=[model_checkpoint],
verbose=1)
def main(argv):
# Define command line options
p = optparse.OptionParser(
description='Pyxit/Cytomine Classification model Builder',
prog='PyXit Classification Model Builder (PYthon piXiT)')
p.add_option(
"--cytomine_host",
type="string",
default='',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('--cytomine_dump_type',
type='int',
dest='cytomine_dump_type',
help="annotation type (1=crop, 2=alphamask)")
# p.add_option('--cytomine_fixed_tile', default=False,action="store_true", dest="cytomine_fixed_tile", help="Force fixed tile size crop around annotations")
p.add_option('--cytomine_fixed_tile',
type="string",
default="False",
dest="cytomine_fixed_tile",
help="Force fixed tile size crop around annotations")
p.add_option(
'--cytomine_n_shifts',
type='int',
dest='cytomine_n_shifts',
help=
"number of translated (shifted) crops extracted for each annotation")
p.add_option('--cytomine_annotation_projects',
type="string",
dest="cytomine_annotation_projects",
help="Projects from which annotations are extracted")
p.add_option('--cytomine_excluded_terms',
type='string',
default='5735',
dest='cytomine_excluded_terms',
help="term ids of excluded terms")
# p.add_option('--cytomine_reviewed', default=False, action="store_true", dest="cytomine_reviewed", help="Get reviewed annotations only")
p.add_option('--cytomine_reviewed',
type='string',
default="False",
dest="cytomine_reviewed",
help="Get reviewed annotations only")
p.add_option('--pyxit_target_width',
type='int',
dest='pyxit_target_width',
help="pyxit subwindows width")
p.add_option('--pyxit_target_height',
type='int',
dest='pyxit_target_height',
help="pyxit subwindows height")
p.add_option(
'--pyxit_colorspace',
type='int',
dest='pyxit_colorspace',
help="pyxit colorspace encoding") # future: get it from server db
p.add_option(
'--pyxit_n_jobs',
type='int',
dest='pyxit_n_jobs',
help="pyxit number of jobs for trees") # future: get it from server db
p.add_option('--pyxit_n_subwindows',
default=10,
type="int",
dest="pyxit_n_subwindows",
help="number of subwindows")
p.add_option('--pyxit_min_size',
default=0.5,
type="float",
dest="pyxit_min_size",
help="min size")
p.add_option('--pyxit_max_size',
default=1.0,
type="float",
dest="pyxit_max_size",
help="max size")
p.add_option('--pyxit_interpolation',
default=2,
type="int",
dest="pyxit_interpolation",
help="interpolation method 1,2,3,4")
# p.add_option('--pyxit_transpose', default=False, action="store_true", dest="pyxit_transpose", help="transpose subwindows")
p.add_option('--pyxit_transpose',
type="string",
default="False",
dest="pyxit_transpose",
help="transpose subwindows")
# p.add_option('--pyxit_fixed_size', default=False, action="store_true", dest="pyxit_fixed_size", help="extract fixed size subwindows")
p.add_option('--pyxit_fixed_size',
type="string",
default="False",
dest="pyxit_fixed_size",
help="extract fixed size subwindows")
p.add_option('--forest_n_estimators',
default=10,
type="int",
dest="forest_n_estimators",
help="number of base estimators (T)")
p.add_option('--forest_max_features',
default=1,
type="int",
dest="forest_max_features",
help="max features at test node (k)")
p.add_option('--forest_min_samples_split',
default=1,
type="int",
dest="forest_min_samples_split",
help="minimum node sample size (nmin)")
p.add_option(
'--svm',
default=0,
dest="svm",
help=
"final svm classifier: 0=nosvm, 1=libsvm, 2=liblinear, 3=lr-l1, 4=lr-l2",
type="int")
p.add_option('--svm_c',
default=1.0,
type="float",
dest="svm_c",
help="svm C")
p.add_option('--cv_k_folds',
default=False,
type="int",
dest="cv_k_folds",
help="number of cross validation folds")
# p.add_option('--cv_shuffle', default=False, action="store_true", dest="cv_shuffle", help="shuffle splits in cross validation")
p.add_option('--cv_shuffle',
type="string",
default="False",
dest="cv_shuffle",
help="shuffle splits in cross validation")
p.add_option('--cv_shuffle_test_fraction',
default=0.3,
type="float",
dest="cv_shuffle_test_fraction",
help="shuffle fraction in cross validation")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=argv)
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_annotation_projects'] = map(
int, options.cytomine_annotation_projects.split(','))
parameters['cytomine_excluded_terms'] = map(
int, options.cytomine_excluded_terms.split(','))
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['cytomine_dump_type'] = options.cytomine_dump_type
parameters['cytomine_fixed_tile'] = str2bool(options.cytomine_fixed_tile)
parameters['cytomine_n_shifts'] = options.cytomine_n_shifts
parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed)
pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width
pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height
pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size
pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size
pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace
pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation
pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators
pyxit_parameters['forest_max_features'] = options.forest_max_features
pyxit_parameters[
'forest_min_samples_split'] = options.forest_min_samples_split
pyxit_parameters['svm'] = options.svm
pyxit_parameters['svm_c'] = options.svm_c
pyxit_parameters['cv_k_folds'] = options.cv_k_folds
pyxit_parameters['cv_shuffle'] = str2bool(options.cv_shuffle)
pyxit_parameters[
'cv_shuffle_test_fraction'] = options.cv_shuffle_test_fraction
pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs
# Check for errors in the options
if options.verbose:
print "[pyxit.main] Options = ", options
# Create JOB/USER/JOB
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
# Create a new userjob if connected as human user
current_user = conn.get_current_user()
run_by_user_job = False
if current_user.algo == False:
print "adduserJob..."
user_job = conn.add_user_job(parameters['cytomine_id_software'],
parameters['cytomine_id_project'])
print "set_credentials..."
conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey))
print "done"
else:
user_job = current_user
print "Already running as userjob"
run_by_user_job = True
job = conn.get_job(user_job.job)
# get annotation collection from Cytomine
annotations = conn.get_annotations(
id_project=parameters['cytomine_id_project'],
reviewed_only=parameters['cytomine_reviewed'])
# set output dir parameters
pyxit_parameters['dir_ls'] = os.path.join(
parameters["cytomine_working_path"],
str(parameters['cytomine_annotation_projects']).replace(
',', '-').replace('[', '').replace(']', '').replace(' ', ''),
"zoom_level", str(parameters['cytomine_zoom_level']), "dump_type",
str(parameters['cytomine_dump_type']))
if not os.path.exists(pyxit_parameters['dir_ls']):
print "Creating annotation directory: %s" % pyxit_parameters['dir_ls']
os.makedirs(pyxit_parameters['dir_ls'])
time.sleep(2)
# image dump type
if (parameters['cytomine_dump_type'] == 1):
annotation_get_func = Annotation.get_annotation_crop_url
elif (parameters['cytomine_dump_type'] == 2):
annotation_get_func = Annotation.get_annotation_alpha_crop_url
else:
print "default annotation type crop"
annotation_get_func = Annotation.get_annotation_crop_url
# dump annotation crops
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Dump annotations...",
progress=50)
annotations = conn.dump_annotations(
annotations=annotations,
get_image_url_func=annotation_get_func,
dest_path=pyxit_parameters['dir_ls'],
desired_zoom=parameters['cytomine_zoom_level'],
excluded_terms=parameters['cytomine_excluded_terms'],
tile_size=parameters['cytomine_fixed_tile'],
translate=parameters['cytomine_n_shifts'])
# build pyxit model(s) and evaluate them (according to cross-validation parameters)
print "Create software parameters values..."
if run_by_user_job == False:
parameters_values = conn.add_job_parameters(
user_job.job,
conn.get_software(parameters['cytomine_id_software']),
pyxit_parameters)
argv = parameters_values_to_argv(pyxit_parameters, parameters_values)
else:
argv = []
for key in pyxit_parameters:
value = pyxit_parameters[key]
if type(value) is bool or value == 'True':
if bool(value):
argv.append("--%s" % key)
elif not value == 'False':
argv.append("--%s" % key)
argv.append("%s" % value)
print "Run PyXiT..."
print argv
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Build models...",
progress=75)
predict = pyxitstandalone.main(argv)
print "------------------- Publishing results to Cytomine Core... ----------------------"
print annotations.data()
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Publishing results...",
progress=90)
for annotation in annotations.data():
# print "ANNOTATION: %s" %annotation
# print "ANNOTATION TERM: %s" %annotation.term
for term in annotation.term:
# annot_descr = conn.get_annotation(annotation.id)
# if hasattr(annotation, "filename"):
# print "filename: %s" %annotation.filename
# time.sleep(1)
if hasattr(annotation, "filename") and (annotation.filename
in predict):
print "PUBLISH annotation %s prediction" % annotation
p = predict[annotation.filename]
annotation_term = AlgoAnnotationTerm()
annotation_term.annotation = annotation.id
annotation_term.expectedTerm = term
annotation_term.term = p[0]
if (pyxit_parameters['svm'] == 1
): # libsvm does not return proba
annotation_term.rate = 1.0
else:
annotation_term.rate = p[1]
conn.add_annotation_term(
annotation.id,
p[0],
term,
annotation_term.rate,
annotation_term_model=AlgoAnnotationTerm)
job = conn.update_job_status(job,
status=job.TERMINATED,
status_comment="Finish",
progress=100)
print "END."
def main():
opt_parser = optparse.OptionParser(description='Cytomine Landmark Detection : Model building', prog='Cytomine Landmark Detector : Model builder', version='0.1')
opt_parser.add_option('--cytomine_host', type="string", default='beta.cytomine.be', dest="cytomine_host", help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
opt_parser.add_option('--cytomine_public_key', type="string", default='XXX', dest="cytomine_public_key", help="Cytomine public key")
opt_parser.add_option('--cytomine_private_key', type="string", default='YYY', dest="cytomine_private_key", help="Cytomine private key")
opt_parser.add_option('--cytomine_id_software', type="int", dest="cytomine_id_software", help="The Cytomine software identifier")
opt_parser.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path", help="Cytomine base path")
opt_parser.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path", help="The working directory (eg: /tmp)")
opt_parser.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project", help="The Cytomine project identifier")
opt_parser.add_option('--cytomine_id_terms', type='string', dest='cytomine_id_terms', help="The identifiers of the terms to create detection models for. Terms must be separated by commas (no spaces). If 'all' is mentioned instead, every terms will be detected.")
opt_parser.add_option('--cytomine_training_images', default='all', type='string', dest='cytomine_training_images', help="identifiers of the images used to create the models. ids must be separated by commas (no spaces). If 'all' is mentioned instead, every image with manual annotation will be used.")
opt_parser.add_option('--image_type', type='string', default='jpg', dest='image_type', help="The type of the images that will be used (jpg, bmp, png,...)")
opt_parser.add_option('--model_njobs', type='int', default=1, dest='model_njobs', help="The number of processors used for model building")
opt_parser.add_option('--model_R', type='int', default=6, dest='model_R', help="Max distance for extracting landmarks")
opt_parser.add_option('--model_RMAX', type='int', default=200, dest='model_RMAX', help="Max distance for extracting non-landmarks")
opt_parser.add_option('--model_P', type='float', default=3, dest='model_P', help="Proportion of non-landmarks")
opt_parser.add_option('--model_npred', type='int', default=50000, dest='model_npred', help="Number of pixels extracted for prediction")
opt_parser.add_option('--model_ntrees', type='int', default=50, dest='model_ntrees', help="Number of trees")
opt_parser.add_option('--model_ntimes', type='int', default=3, dest='model_ntimes', help="Number of rotations to apply to the image")
opt_parser.add_option('--model_angle', type='float', default=30, dest='model_angle', help="Max angle for rotation")
opt_parser.add_option('--model_depth', type='int', default=5, dest='model_depth', help="Number of resolutions to use")
opt_parser.add_option('--model_step', type='int', default=1, dest='model_step', help="Landmark pixels will be extracted in a grid (x-R:step:x+r,y-R:step:y+R) around the landmark")
opt_parser.add_option('--model_wsize', type='int', default=8, dest='model_wsize', help="Window size")
opt_parser.add_option('--model_feature_type', type='string', default='haar', dest='model_feature_type', help='The type of feature (raw, sub, haar or gaussian).')
opt_parser.add_option('--model_feature_haar_n', type='int', default=1600, dest='model_feature_haar_n', help='Haar-Like features only. Number of descriptors for a pixel. Must be a multiple of 5*depths.')
opt_parser.add_option('--model_feature_gaussian_n', type='int', default=1600, dest='model_feature_gaussian_n', help='Gaussian features only. Number of descriptors for a pixel. Must be a multiple of depths.')
opt_parser.add_option('--model_feature_gaussian_std', type='float', default=20., dest='model_feature_gaussian_std', help='Gaussian features only. Standard deviation for the gaussian.')
opt_parser.add_option('--model_save_to', type='string', default='/tmp/', dest='model_save_to', help="Destination for model storage")
opt_parser.add_option('--model_name', type='string', dest='model_name', help="Name of the model (used for saving)")
opt_parser.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode")
opts, argus = opt_parser.parse_args(args=sys.argv)
conn = cytomine.Cytomine(opts.cytomine_host, opts.cytomine_public_key, opts.cytomine_private_key, base_path=opts.cytomine_base_path, working_path=opts.cytomine_working_path, verbose=str2bool(opts.verbose))
current_user = conn.get_current_user()
if not current_user.algo:
user_job = conn.add_user_job(opts.cytomine_id_software, opts.cytomine_id_project)
conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey))
else:
user_job = current_user
job = conn.get_job(user_job.job)
job = conn.update_job_status(job, status=job.RUNNING, progress=0, status_comment="Bulding model...")
job_parameters = {'cytomine_id_terms': opts.cytomine_id_terms, 'cytomine_training_images': opts.cytomine_training_images, 'model_R': opts.model_R, 'model_njobs': opts.model_njobs, 'model_RMAX': opts.model_RMAX, 'model_P': opts.model_P, 'model_npred': opts.model_npred, 'model_ntimes': opts.model_ntimes, 'model_angle': opts.model_angle, 'model_depth': opts.model_depth, 'model_wsize': opts.model_wsize, 'model_ntrees': opts.model_ntrees, 'model_step': opts.model_step, 'forest_max_features': ((2 * opts.model_wsize) ** 2) * opts.model_depth, 'forest_min_samples_split': 2, 'model_name': opts.model_name, 'model_feature_type': opts.model_feature_type, 'model_feature_haar_n': opts.model_feature_haar_n,
'model_feature_gaussian_n': opts.model_feature_gaussian_n, 'model_feature_gaussian_std': opts.model_feature_gaussian_std}
conn.add_job_parameters(user_job.job, conn.get_software(opts.cytomine_id_software), job_parameters)
download_images(conn, opts.cytomine_id_project)
download_annotations(conn, opts.cytomine_id_project, opts.cytomine_working_path)
repository = opts.cytomine_working_path + str(opts.cytomine_id_project) + '/'
txt_repository = opts.cytomine_working_path + '%d/txt/' % opts.cytomine_id_project
depths = 1. / (2. ** np.arange(opts.model_depth))
(xc, yc, xr, yr, ims, t_to_i, i_to_t) = getallcoords(txt_repository)
if opts.cytomine_id_terms == 'all':
term_list = t_to_i.keys()
else:
term_list = [int(term) for term in opts.cytomine_id_terms.split(',')]
if opts.cytomine_training_images == 'all':
tr_im = ims
else:
tr_im = [int(id_im) for id_im in opts.cytomine_training_images.split(',')]
DATA = None
REP = None
be = 0
model_repo = opts.model_save_to
for id_term in term_list:
(xc, yc, xr, yr) = getcoordsim(txt_repository, id_term, tr_im)
nimages = np.max(xc.shape)
mx = np.mean(xr)
my = np.mean(yr)
P = np.zeros((2, nimages))
P[0, :] = xr
P[1, :] = yr
cm = np.cov(P)
passe = False
progress = 0
delta = 80 / opts.model_ntimes
# additional parameters
feature_parameters = None
if opts.model_feature_type.lower() == 'gaussian':
std_matrix = np.eye(2) * (opts.model_feature_gaussian_std ** 2)
feature_parameters = np.round(np.random.multivariate_normal([0, 0], std_matrix, opts.model_feature_gaussian_n)).astype(int)
elif opts.model_feature_type.lower() == 'haar':
W = opts.model_wsize
n = opts.model_feature_haar_n / (5 * opts.model_depth)
h2 = generate_2_horizontal(W, n)
v2 = generate_2_vertical(W, n)
h3 = generate_3_horizontal(W, n)
v3 = generate_3_vertical(W, n)
sq = generate_square(W, n)
feature_parameters = (h2, v2, h3, v3, sq)
for times in range(opts.model_ntimes):
if times == 0:
rangrange = 0
else:
rangrange = opts.model_angle
T = build_datasets_rot_mp(repository, tr_im, xc, yc, opts.model_R, opts.model_RMAX, opts.model_P, opts.model_step, rangrange, opts.model_wsize, opts.model_feature_type, feature_parameters, depths, nimages, opts.image_type, opts.model_njobs)
for i in range(len(T)):
(data, rep, img) = T[i]
(height, width) = data.shape
if not passe:
passe = True
DATA = np.zeros((height * (len(T) + 100) * opts.model_ntimes, width))
REP = np.zeros(height * (len(T) + 100) * opts.model_ntimes)
b = 0
be = height
DATA[b:be, :] = data
REP[b:be] = rep
b = be
be = be + height
progress += delta
job = conn.update_job_status(job, status=job.RUNNING, status_comment="Bulding model...", progress=progress)
REP = REP[0:b]
DATA = DATA[0:b, :]
clf = ExtraTreesClassifier(n_jobs=opts.model_njobs, n_estimators=opts.model_ntrees)
clf = clf.fit(DATA, REP)
job = conn.update_job_status(job, status=job.RUNNING, progress=90, status_comment="Writing model...")
if not os.path.isdir(model_repo):
os.mkdir(model_repo)
joblib.dump(clf, '%s%s_%d.pkl' % (model_repo, opts.model_name, id_term))
joblib.dump([mx, my, cm], '%s%s_%d_cov.pkl' % (model_repo, opts.model_name, id_term))
if opts.model_feature_type == 'haar' or opts.model_feature_type == 'gaussian':
joblib.dump(feature_parameters, '%s%s_%d_fparameters.pkl' % (model_repo, opts.model_name, id_term))
f = open('%s%s.conf' % (model_repo, opts.model_name), 'w+')
f.write('cytomine_id_terms %s\n' % opts.cytomine_id_terms)
f.write('model_R %d\n' % opts.model_R)
f.write('model_RMAX %d\n' % opts.model_RMAX)
f.write('model_P %f\n' % opts.model_P)
f.write('model_npred %d\n' % opts.model_npred)
f.write('model_ntrees %d\n' % opts.model_ntrees)
f.write('model_ntimes %d\n' % opts.model_ntimes)
f.write('model_angle %f\n' % opts.model_angle)
f.write('model_depth %d\n' % opts.model_depth)
f.write('model_step %d\n' % opts.model_step)
f.write('window_size %d\n' % opts.model_wsize)
f.write('feature_type %s\n' % opts.model_feature_type)
f.write('feature_haar_n %d\n' % opts.model_feature_haar_n)
f.write('feature_gaussian_n %d\n' % opts.model_feature_gaussian_n)
f.write('feature_gaussian_std %f' % opts.model_feature_gaussian_std)
f.close()
conn.update_job_status(job, status=job.TERMINATED, progress=100, status_comment="Model built!")
def main(argv):
# Define command line options
p = optparse.OptionParser(
description='Pyxit/Cytomine Segmentation Model Builder',
prog='PyXit Segmentation Model Builder (PYthon piXiT)')
p.add_option(
"--cytomine_host",
type="string",
default='',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('--cytomine_annotation_projects',
type="string",
dest="cytomine_annotation_projects",
help="Projects from which annotations are extracted")
p.add_option(
'--cytomine_predict_terms',
type='string',
default='0',
dest='cytomine_predict_terms',
help="term ids of predicted terms (=positive class in binary mode)")
p.add_option('--cytomine_excluded_terms',
type='string',
default='0',
dest='cytomine_excluded_terms',
help="term ids of excluded terms")
p.add_option('--cytomine_reviewed',
type='string',
default="False",
dest="cytomine_reviewed",
help="Get reviewed annotations only")
p.add_option('--pyxit_target_width',
type='int',
dest='pyxit_target_width',
help="pyxit subwindows width")
p.add_option('--pyxit_target_height',
type='int',
dest='pyxit_target_height',
help="pyxit subwindows height")
p.add_option('--pyxit_save_to',
type='string',
dest='pyxit_save_to',
help="pyxit model directory") #future: get it from server db
p.add_option(
'--pyxit_colorspace',
type='int',
dest='pyxit_colorspace',
help="pyxit colorspace encoding") #future: get it from server db
p.add_option(
'--pyxit_n_jobs',
type='int',
dest='pyxit_n_jobs',
help="pyxit number of jobs for trees") #future: get it from server db
p.add_option('--pyxit_n_subwindows',
default=10,
type="int",
dest="pyxit_n_subwindows",
help="number of subwindows")
p.add_option('--pyxit_interpolation',
default=2,
type="int",
dest="pyxit_interpolation",
help="interpolation method 1,2,3,4")
p.add_option('--pyxit_transpose',
type="string",
default="False",
dest="pyxit_transpose",
help="transpose subwindows")
p.add_option('--pyxit_fixed_size',
type="string",
default="False",
dest="pyxit_fixed_size",
help="extract fixed size subwindows")
p.add_option('--forest_n_estimators',
default=10,
type="int",
dest="forest_n_estimators",
help="number of base estimators (T)")
p.add_option('--forest_max_features',
default=1,
type="int",
dest="forest_max_features",
help="max features at test node (k)")
p.add_option('--forest_min_samples_split',
default=1,
type="int",
dest="forest_min_samples_split",
help="minimum node sample size (nmin)")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=argv)
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_annotation_projects'] = map(
int, options.cytomine_annotation_projects.split(','))
parameters['cytomine_predict_terms'] = map(
int, options.cytomine_predict_terms.split(','))
parameters['cytomine_excluded_terms'] = map(
int, options.cytomine_excluded_terms.split(','))
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed)
pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width
pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height
pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace
pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation
pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators
pyxit_parameters['forest_max_features'] = options.forest_max_features
pyxit_parameters[
'forest_min_samples_split'] = options.forest_min_samples_split
pyxit_parameters['pyxit_save_to'] = options.pyxit_save_to
pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs
# Check for errors in the options
if options.verbose:
print "[pyxit.main] Options = ", options
# Create JOB/USER/JOB
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
#Create a new userjob if connected as human user
current_user = conn.get_current_user()
if current_user.algo == False:
print "adduserJob..."
user_job = conn.add_user_job(parameters['cytomine_id_software'],
parameters['cytomine_id_project'])
print "set_credentials..."
conn.set_credentials(str(user_job.publicKey), str(user_job.privateKey))
print "done"
else:
user_job = current_user
print "Already running as userjob"
job = conn.get_job(user_job.job)
pyxit_parameters['dir_ls'] = os.path.join(
parameters["cytomine_working_path"],
str(parameters['cytomine_annotation_projects']).replace(
',', '-').replace('[', '').replace(']', '').replace(' ', ''),
"zoom_level", str(parameters['cytomine_zoom_level']))
if not os.path.exists(pyxit_parameters['dir_ls']):
print "Creating annotation directory: %s" % pyxit_parameters['dir_ls']
os.makedirs(pyxit_parameters['dir_ls'])
time.sleep(2)
job = conn.update_job_status(
job, status_comment="Publish software parameters values")
all_params = pyxit_parameters
all_params.update(parameters)
job_parameters_values = conn.add_job_parameters(
user_job.job, conn.get_software(parameters['cytomine_id_software']),
all_params)
#Get annotation data
job = conn.update_job_status(job,
status=job.RUNNING,
status_comment="Fetching data",
progress=0)
#Retrieve annotations from each annotation projects, either reviewed or unreviewed annotations
annotations = None
for prj in parameters['cytomine_annotation_projects']:
if parameters["cytomine_reviewed"]:
print "Retrieving reviewed annotations..."
annotations_prj = conn.get_annotations(id_project=prj,
reviewed_only=True)
print "Reviewed annotations: %d" % len(annotations_prj.data())
else:
print "Retrieving (unreviewed) annotations..."
annotations_prj = conn.get_annotations(id_project=prj)
print "(Unreviewed) annotations: %d" % len(annotations_prj.data())
if not annotations:
annotations = annotations_prj
else:
annotations.data().extend(annotations_prj.data())
print "Nb annotations so far... = %d" % len(annotations.data())
time.sleep(3)
print "Total annotations projects %s = %d" % (
parameters['cytomine_annotation_projects'], len(annotations.data()))
time.sleep(3)
print "Predict terms / excluded terms"
print parameters['cytomine_predict_terms']
print parameters['cytomine_excluded_terms']
time.sleep(3)
annotations = conn.dump_annotations(
annotations=annotations,
get_image_url_func=Annotation.get_annotation_alpha_crop_url,
dest_path=pyxit_parameters['dir_ls'],
excluded_terms=parameters['cytomine_excluded_terms'],
desired_zoom=parameters['cytomine_zoom_level'])
#Build matrix (subwindows described by pixel values and output) for training
project = conn.get_project(parameters['cytomine_id_project'])
terms = conn.get_terms(project.ontology)
map_classes = {
} # build X, Y. Change initial problem into binary problem : "predict_terms" vs others
for term in terms.data():
if term.id in parameters['cytomine_predict_terms']:
map_classes[term.id] = 1
else:
map_classes[term.id] = 0
print pyxit_parameters
#Prepare image matrix
X, y = build_from_dir(pyxit_parameters['dir_ls'], map_classes)
print "X length: %d " % len(X)
print "Y length: %d " % len(y)
time.sleep(5)
#classes = np.unique(y)
classes = [0, 1]
n_classes = len(classes)
y_original = y
y = np.searchsorted(classes, y)
# Instantiate classifiers
job = conn.update_job_status(
job,
status=job.RUNNING,
status_comment="[pyxit.main] Initializing PyxitClassifier...",
progress=25)
forest = ExtraTreesClassifier(
n_estimators=pyxit_parameters['forest_n_estimators'],
max_features=pyxit_parameters['forest_max_features'],
min_samples_split=pyxit_parameters['forest_min_samples_split'],
n_jobs=pyxit_parameters['pyxit_n_jobs'],
verbose=True)
pyxit = PyxitClassifier(
base_estimator=forest,
n_subwindows=pyxit_parameters['pyxit_n_subwindows'],
min_size=0.0, #segmentation use fixed-size subwindows
max_size=1.0, #segmentation use fixed-size subwindows
target_width=pyxit_parameters['pyxit_target_width'],
target_height=pyxit_parameters['pyxit_target_height'],
interpolation=pyxit_parameters['pyxit_interpolation'],
transpose=pyxit_parameters['pyxit_transpose'],
colorspace=pyxit_parameters['pyxit_colorspace'],
fixed_size=pyxit_parameters['pyxit_fixed_size'],
n_jobs=pyxit_parameters['pyxit_n_jobs'],
verbose=True,
get_output=_get_output_from_mask)
if pyxit_parameters['pyxit_save_to']:
d = os.path.dirname(pyxit_parameters['pyxit_save_to'])
if not os.path.exists(d):
os.makedirs(d)
fd = open(pyxit_parameters['pyxit_save_to'], "wb")
pickle.dump(classes, fd, protocol=pickle.HIGHEST_PROTOCOL)
job = conn.update_job_status(
job,
status_comment=
"[pyxit.main] Extracting %d subwindows from each image in %s" %
(pyxit_parameters['pyxit_n_subwindows'], pyxit_parameters['dir_ls']),
progress=50)
time.sleep(3)
#Extract random subwindows in dumped annotations
_X, _y = pyxit.extract_subwindows(X, y)
#Build pixel classifier
job = conn.update_job_status(
job,
status_comment="[pyxit.main] Fitting Pyxit Segmentation Model on %s",
progress=75)
print "TIME : %s" % strftime("%Y-%m-%d %H:%M:%S", localtime())
start = time.time()
pyxit.fit(X, y, _X=_X, _y=_y)
end = time.time()
print "Elapsed time FIT: %d s" % (end - start)
print "TIME : %s" % strftime("%Y-%m-%d %H:%M:%S", localtime())
print "pyxit.base_estimator.n_classes_"
print pyxit.base_estimator.n_classes_
print "pyxit.base_estimator.classes_"
print pyxit.base_estimator.classes_
if options.verbose:
print "----------------------------------------------------------------"
print "[pyxit.main] Saving Pyxit Segmentation Model locally into %s" % pyxit_parameters[
'pyxit_save_to']
print "----------------------------------------------------------------"
#Save model on local disk
if pyxit_parameters['pyxit_save_to']:
pickle.dump(pyxit, fd, protocol=pickle.HIGHEST_PROTOCOL)
if pyxit_parameters['pyxit_save_to']:
fd.close()
print "Not Publishing model in db.."
#job_data = conn.add_job_data(job, "model", pyxit_parameters['pyxit_save_to'])
job = conn.update_job_status(job,
status=job.TERMINATED,
status_comment="Finish",
progress=100)
print "END."
def main(argv):
# Parameter values are set through command-line, see test-train.sh
parameters = {
'cytomine_host': None,
'cytomine_public_key': None,
'cytomine_private_key': None,
'cytomine_base_path': None,
'cytomine_working_path': '/home/maree/tmp/cytomine/annotations/',
'cytomine_id_software': 1,
'cytomine_id_project': 1,
'cytomine_zoom_level': 1,
'cytomine_dump_type': 1,
'cytomine_annotation_projects':
[1], # id of projets from which we dump annotations for learning
'cytomine_predict_terms': [1], #
'cytomine_excluded_terms': [2, 3], # exclude these term ids
'cytomine_reviewed': True
}
# Parameter values are set through command-line, see test-train.sh
pyxit_parameters = {
'dir_ls': "/",
# 'dir_ts' : "/",
'forest_shared_mem': False,
# processing
'pyxit_n_jobs': 10,
# subwindows extraction
'pyxit_n_subwindows': 100,
'pyxit_min_size': 0.1,
'pyxit_max_size': 1.0,
'pyxit_target_width': 24, # 24x24
'pyxit_target_height': 24,
'pyxit_interpolation': 1,
'pyxit_transpose':
1, # do we apply rotation/mirroring to subwindows (to enrich training set)
'pyxit_colorspace': 2, # which colorspace do we use ?
'pyxit_fixed_size':
False, # do we extracted fixed sizes or random sizes (false)
# classifier parameters
'forest_n_estimators': 10, # number of trees
'forest_max_features':
28, # number of attributes considered at each node
'forest_min_samples_split': 1, # nmin
'svm': 0,
'svm_c': 1.0,
}
# Define command line options
p = optparse.OptionParser(
description='Pyxit/Cytomine Classification Model Builder',
prog='PyXit Classification Model Builder (PYthon piXiT)')
# Cytomine arguments
p.add_option(
"--cytomine_host",
type="string",
default='',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('--cytomine_dump_type',
type='int',
dest='cytomine_dump_type',
help="annotation type (1=crop, 2=alphamask)")
p.add_option('--cytomine_annotation_projects',
type="string",
dest="cytomine_annotation_projects",
help="Projects from which annotations are extracted")
p.add_option(
'--cytomine_predict_terms',
type='string',
default='0',
dest='cytomine_predict_terms',
help="term ids of predicted terms (=positive class in binary mode)")
p.add_option('--cytomine_excluded_terms',
type='string',
default='0',
dest='cytomine_excluded_terms',
help="term ids of excluded terms")
p.add_option('--cytomine_reviewed',
type='string',
default="False",
dest="cytomine_reviewed",
help="Get reviewed annotations only")
p.add_option('--path_to_project_info',
type="string",
default=None,
dest="path_to_project_info",
help="Path to file containing the project information")
p.add_option('--cytomine_test_projects',
type="string",
dest="cytomine_test_projects",
help="Projects used to test the classification models")
p.add_option('--cytomine_n_test_images_per_project',
type="int",
dest="cytomine_n_test_images_per_project",
help="Number of images to test on per project")
p.add_option('--build_models',
type="string",
default="0",
dest="build_models",
help="Turn on (1) or off (0) model building")
p.add_option('--test_models',
type="string",
default="0",
dest="test_models",
help="Turn on (1) or off (0) model testing")
p.add_option('--cytomine_dump',
type="string",
default="0",
dest="cytomine_dump",
help="Turn on (1) or off (0) model annotation dump")
p.add_option('--dump_test_annotations',
type="string",
default="0",
dest="dump_test_annotations",
help="Turn on (1) or off (0) test annotation dump")
p.add_option('--plot_results',
type="string",
default="0",
dest="plot_results",
help="Turn on (1) or off (0) result plot")
# Pyxit arguments
p.add_option('--pyxit_target_width',
type='int',
dest='pyxit_target_width',
help="pyxit subwindows width")
p.add_option('--pyxit_target_height',
type='int',
dest='pyxit_target_height',
help="pyxit subwindows height")
p.add_option('--pyxit_save_to',
type='string',
dest='pyxit_save_to',
help="pyxit model directory") # future: get it from server db
p.add_option(
'--pyxit_colorspace',
type='int',
dest='pyxit_colorspace',
help="pyxit colorspace encoding") # future: get it from server db
p.add_option(
'--pyxit_n_jobs',
type='int',
dest='pyxit_n_jobs',
help="pyxit number of jobs for trees") # future: get it from server db
p.add_option('--pyxit_n_subwindows',
default=10,
type="int",
dest="pyxit_n_subwindows",
help="number of subwindows")
p.add_option('--pyxit_min_size',
default=0.5,
type="float",
dest="pyxit_min_size",
help="min size")
p.add_option('--pyxit_max_size',
default=1.0,
type="float",
dest="pyxit_max_size",
help="max size")
p.add_option('--pyxit_interpolation',
default=2,
type="int",
dest="pyxit_interpolation",
help="interpolation method 1,2,3,4")
p.add_option('--pyxit_transpose',
type="string",
default="False",
dest="pyxit_transpose",
help="transpose subwindows")
p.add_option('--pyxit_fixed_size',
type="string",
default="False",
dest="pyxit_fixed_size",
help="extract fixed size subwindows")
p.add_option('--cv_k_folds',
type="int",
dest="cv_k_folds",
help="The number of folds")
p.add_option('--forest_n_estimators',
default=10,
type="int",
dest="forest_n_estimators",
help="number of base estimators (T)")
p.add_option('--forest_max_features',
default=1,
type="int",
dest="forest_max_features",
help="max features at test node (k)")
p.add_option('--forest_min_samples_split',
default=1,
type="int",
dest="forest_min_samples_split",
help="minimum node sample size (nmin)")
p.add_option('--forest_shared_mem',
default=False,
action="store_true",
dest="forest_shared_mem",
help="shared mem")
p.add_option(
'--svm',
default=0,
dest="svm",
help=
"final svm classifier: 0=nosvm, 1=libsvm, 2=liblinear, 3=lr-l1, 4=lr-l2",
type="int")
p.add_option('--svm_c',
default=1.0,
type="float",
dest="svm_c",
help="svm C")
# p.add_option('--verbose', action="store_true", default=True, dest="verbose", help="Turn on verbose mode")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=argv)
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_annotation_projects'] = map(
int, options.cytomine_annotation_projects.split(','))
parameters['cytomine_test_projects'] = map(
int, options.cytomine_test_projects.split(','))
parameters['cytomine_predict_terms'] = map(
int, options.cytomine_predict_terms.split(','))
parameters['cytomine_excluded_terms'] = map(
int, options.cytomine_excluded_terms.split(','))
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['cytomine_dump_type'] = options.cytomine_dump_type
parameters['cytomine_dump'] = str2bool(options.cytomine_dump)
parameters['test_models'] = str2bool(options.test_models)
parameters['build_models'] = str2bool(options.build_models)
parameters['dump_test_annotations'] = str2bool(
options.dump_test_annotations)
parameters['plot_results'] = str2bool(options.plot_results)
parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed)
parameters['path_to_project_info'] = options.path_to_project_info
parameters[
'cytomine_n_test_images_per_project'] = options.cytomine_n_test_images_per_project
pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width
pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height
pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size
pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size
pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace
pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation
pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
pyxit_parameters['cv_k_folds'] = options.cv_k_folds
pyxit_parameters['forest_n_estimators'] = options.forest_n_estimators
pyxit_parameters['forest_max_features'] = options.forest_max_features
pyxit_parameters[
'forest_min_samples_split'] = options.forest_min_samples_split
pyxit_parameters['forest_shared_mem'] = options.forest_min_samples_split
pyxit_parameters['svm'] = options.svm
pyxit_parameters['svm_c'] = options.svm_c
pyxit_parameters['pyxit_save_to'] = options.pyxit_save_to
pyxit_parameters['pyxit_n_jobs'] = options.pyxit_n_jobs
pyxit_parameters['dir_ls'] = os.path.join(
parameters["cytomine_working_path"],
str(parameters['cytomine_annotation_projects']).replace(
',', '-').replace('[', '').replace(']', '').replace(' ', ''),
"zoom_level", str(parameters['cytomine_zoom_level']), "dump_type",
str(parameters['cytomine_dump_type']))
# Check for errors in the options
if options.verbose:
print "[pyxit.main] Options = ", options
# Create JOB/USER/JOB
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
# Image dump type (for classification use 1)
if (parameters['cytomine_dump_type'] == 1):
annotation_get_func = Annotation.get_annotation_crop_url
elif (parameters['cytomine_dump_type'] == 2):
annotation_get_func = Annotation.get_annotation_alpha_crop_url
else:
print "default annotation type crop"
annotation_get_func = Annotation.get_annotation_crop_url
if parameters['cytomine_dump']:
# Get annotation descriptions (JSON) from project(s)
annotations = None
for prj in parameters['cytomine_annotation_projects']:
if parameters["cytomine_reviewed"]:
print "Retrieving reviewed annotations..."
annotations_prj = conn.get_annotations(
id_project=prj,
reviewed_only=parameters["cytomine_reviewed"])
print "Reviewed annotations: %d" % len(annotations_prj.data())
else: # We go here
print "Retrieving (unreviewed) annotations..."
annotations_prj = conn.get_annotations(id_project=prj)
print "(Unreviewed) annotations: %d" % len(
annotations_prj.data())
if not annotations:
annotations = annotations_prj
else:
annotations.data().extend(annotations_prj.data())
if prj == 21907448 or prj == 155194683:
annotations_prj = conn.get_annotations(id_project=prj,
id_term=91376951)
print "Notadeno annotations: %d" % len(annotations_prj.data())
annotations.data().extend(annotations_prj.data())
print "Nb annotations so far... = %d" % len(annotations.data())
print "Total annotations projects %s = %d" % (
parameters['cytomine_annotation_projects'], len(
annotations.data()))
# Set output dir parameters
if not os.path.exists(pyxit_parameters['dir_ls']):
print "Creating annotation directory: %s" % pyxit_parameters[
'dir_ls']
os.makedirs(pyxit_parameters['dir_ls'])
# Dump annotation images locally
conn.dump_annotations(
annotations=annotations,
get_image_url_func=annotation_get_func,
dest_path=pyxit_parameters['dir_ls'],
desired_zoom=parameters['cytomine_zoom_level'],
excluded_terms=parameters['cytomine_excluded_terms'])
quit()
# Put positive terms under the same term and same for negative terms
term_directories = os.listdir(pyxit_parameters['dir_ls'])
pos_path = os.path.join(pyxit_parameters['dir_ls'], "1")
if not os.path.exists(pos_path):
print "Creating positive annotation directory: %s" % pos_path
os.makedirs(pos_path)
neg_path = os.path.join(pyxit_parameters['dir_ls'], "0")
if not os.path.exists(neg_path):
print "Creating negative annotation directory: %s" % neg_path
os.makedirs(neg_path)
print parameters['cytomine_predict_terms']
for dir in term_directories:
dir_abs = os.path.join(pyxit_parameters['dir_ls'], dir)
# Move files
if int(dir) in parameters['cytomine_predict_terms']:
for image_file in os.listdir(dir_abs):
os.rename(os.path.join(dir_abs, image_file),
os.path.join(pos_path, image_file))
else:
for image_file in os.listdir(dir_abs):
os.rename(os.path.join(dir_abs, image_file),
os.path.join(neg_path, image_file))
# Remove empty directory
os.rmdir(dir_abs)
print pyxit_parameters['dir_ls']
os.path.dirname(pyxit_parameters['dir_ls'])
# Put negative terms undre the same term
print "END dump annotations"
# Build models
if parameters['build_models']:
# Create directory
if pyxit_parameters['cv_k_folds'] is None:
d = os.path.dirname(pyxit_parameters['pyxit_save_to'])
if not os.path.exists(d):
os.makedirs(d)
# Build models with varying parameters
min_size = 0.1
for max_size in np.arange(0.15, 0.8, 0.05):
# Model without svm
build_model(pyxit_parameters,
parameters,
min_size,
max_size,
n_trees=100,
svm=0,
min_samples_split=10)
quit()
# Model with svm
build_model(pyxit_parameters,
parameters,
min_size,
max_size,
n_trees=100,
svm=1,
min_samples_split=100)
# Dump test annotations
if parameters['dump_test_annotations']:
# Dump test project images
image_folder_records = open("image_folders_record.csv", 'w')
annotations = None
for project_id in parameters['cytomine_test_projects']:
# Get userjob_id to test for each image
if parameters['path_to_project_info'] is not None:
path = os.path.join(parameters["path_to_project_info"],
str(project_id))
df = pd.read_csv(os.path.join(path, "image_info.csv"), sep=';')
images = df.loc[df['Term ID'] == 20202][np.isfinite(
df['Job user ID'])].as_matrix(['Image ID', 'Job user ID'])
else:
images = np.array([
parameters["cytomine_id_image"],
parameters['cytomine_id_userjob']
])
# Select random images from project
n_images = len(images)
n_test_images_per_project = parameters[
'cytomine_n_test_images_per_project']
if n_images < parameters['cytomine_n_test_images_per_project']:
n_test_images_per_project = n_images
if n_images == 0 and n_test_images_per_project == 0:
continue
random_idx = np.random.choice(n_images,
n_test_images_per_project,
replace=False)
# For each randomly selected image,
for idx in random_idx:
i_id = int(images[idx][0])
u_id = int(images[idx][1])
candidate_annotations = conn.get_annotations(
id_user=u_id, id_image=i_id, id_project=project_id)
print "Number of annotations to predict: %d" % len(
candidate_annotations.data())
folder_name = os.path.join(
parameters["cytomine_working_path"], "annotations",
"project-{}".format(project_id),
"crops-candidates-{}-{}".format(i_id, u_id),
"zoom-{}".format(parameters['cytomine_zoom_level']))
if not os.path.exists(folder_name):
os.makedirs(folder_name)
print "Dumping annotation cropped images to classify to %s" % folder_name
# Dump annotation images locally
conn.dump_annotations(
annotations=candidate_annotations,
get_image_url_func=annotation_get_func,
dest_path=folder_name,
desired_zoom=parameters['cytomine_zoom_level'])
# Record image in file
image_folder_records.write(folder_name + "\n")
image_folder_records.close()
# Test each model with each test image
if parameters['test_models']:
# Open folder with where the annotation images are
with open("image_folders_record.csv", 'r') as f:
image_folders = f.readlines()
model_folder = pyxit_parameters['pyxit_save_to']
print "Open model folder :"
print model_folder
for model_name in sorted(os.listdir(model_folder)):
print "Open model name :"
print model_name
# Create model test result folder
test_results_folder = os.path.join(
parameters['cytomine_working_path'],
"test_results_colorspace1", model_name)
if not os.path.exists(test_results_folder):
os.makedirs(test_results_folder)
for model in sorted(
os.listdir(os.path.join(model_folder, model_name))):
# Record model test results
test_result_file_name = model.strip('.pkl') + '.csv'
test_result_file = open(
os.path.join(test_results_folder, test_result_file_name),
'w')
test_result_file.write("Project ID;"
"Image ID;"
"Userjob ID;"
"Annotation ID;"
"Reviewed;"
"Prediction;"
"Positive prediction probability;"
"Negative prediction probability\n")
# Load Classifier model
print "Open model :"
print model
model_path = os.path.join(model_folder, model_name, model)
classifier = open(model_path, "r")
classes = pickle.load(classifier)
pyxit = pickle.load(classifier)
print "Model: %s" % pyxit
# Retrieve model information
annotation_projects, rest = model.split("_svm")
tmp, rest = rest.split("_minsize")
svm = int(tmp)
tmp, rest = rest.split("%_maxsize")
min_size = int(tmp) / 100
tmp, rest = rest.split("%_ntrees")
max_size = int(tmp) / 100
tmp, rest = rest.split(".pkl")
n_trees = int(tmp)
for image_folder_name in image_folders:
image_folder_name = image_folder_name.strip("\n")
# Get image annotation information
zoom = os.path.basename(image_folder_name).strip("zoom-")
rest = os.path.dirname(image_folder_name)
candidate_name = os.path.basename(rest).strip(
"crops-candidates-")
image_id, userjob_id = candidate_name.split("-")
image_id = int(image_id)
userjob_id = int(userjob_id)
rest = os.path.dirname(rest)
project_id = int(os.path.basename(rest).strip("project-"))
# Get reviewed annotations from image
terms = str(parameters['cytomine_predict_terms']).strip(
'[').strip(']').replace(' ', '')
annotations = conn.get_annotations(id_project=project_id,
id_image=image_id,
id_term=terms,
showWKT=True,
showMeta=True,
showGIS=True,
reviewed_only=True)
# Get parent id of reviewed annotations (Annotations whose ID is a key in the table is reviewed)
review_parent_id_table = {}
for annot in annotations.data():
# Get parent id and put (parent_id, reviewed_id) in a table
review_parent_id_table[annot.parentIdent] = annot.id
print "Building subwindows from ", image_folder_name
# Extract subwindows from all candidates annotations
X, y = pyxitstandalone.build_from_dir(image_folder_name)
# Apply pyxit classifier model to X (parameters are already reused from model pkl file)
y_proba = pyxit.predict_proba(X) * 100
for j, annotation_file_name in enumerate(X):
print annotation_file_name
annotation_id = int(
os.path.basename(annotation_file_name).strip(
'.png').split('_')[1])
y_predict = classes.take(np.argmax(y_proba[j], 0),
axis=0)
y_rate = np.max(y_proba[j])
# Check if annotation has been reviewed
reviewed = False
if review_parent_id_table.has_key(annotation_id):
reviewed = True
test_result_file.write(
"{};{};{};{};{};{};{};{}\n".format(
project_id, image_id, userjob_id,
annotation_id, reviewed, y_predict,
y_proba[j, 1], y_proba[j, 0]))
test_result_file.close()
# Plot results
if parameters['plot_results']:
test_results_folder = os.path.join(parameters['cytomine_working_path'],
"test_results_colorspace0")
stats_file = open(
os.path.join(parameters['cytomine_working_path'], 'stats.txt'),
'w') # file containing test stats
for sub_folder in sorted(os.listdir(test_results_folder)):
for results_file in sorted(
os.listdir(os.path.join(test_results_folder, sub_folder))):
# Read results file
print os.path.join(test_results_folder, sub_folder,
results_file)
# Read csv
df = pd.read_csv(os.path.join(test_results_folder, sub_folder,
results_file),
sep=';')
df.insert(0, "Index", df.index)
n_data = df.shape[0]
model = os.path.join(sub_folder,
results_file.strip('.csv') + '.plk')
stats_file.write('\n\nModel name : {}\n'.format(model))
# Find unique project ids
plt.figure(figsize=[14, 7])
unique_project_ids = df['Project ID'].unique()
n_projects = len(unique_project_ids)
# Get annotation prediciton probability
pred_positive = df.loc[df['Reviewed'] == True].as_matrix(
["Index", "Positive prediction probability"])
pred_negative = df.loc[df['Reviewed'] == False].as_matrix(
["Index", "Positive prediction probability"])
precision_array = np.zeros(n_projects)
recall_array = np.zeros(n_projects)
tp_array = np.zeros(n_projects)
fn_array = np.zeros(n_projects)
fp_array = np.zeros(n_projects)
tn_array = np.zeros(n_projects)
for i, id in enumerate(unique_project_ids):
project_data = df.loc[df['Project ID'] == id]
idx_id = int(project_data['Index'].head(1))
plt.axvline(x=idx_id,
color='b',
linestyle='dashed',
linewidth=1,
alpha=0.3)
# Write stats about model tests
project_name = conn.get_project(id).name
print project_name
tp_array[i] = project_data.loc[
project_data['Reviewed'] == True].loc[
project_data['Prediction'] == 1].shape[0]
fn_array[i] = project_data.loc[
project_data['Reviewed'] == True].loc[
project_data['Prediction'] == 0].shape[0]
fp_array[i] = project_data.loc[
project_data['Reviewed'] == False].loc[
project_data['Prediction'] == 1].shape[0]
tn_array[i] = project_data.loc[
project_data['Reviewed'] == False].loc[
project_data['Prediction'] == 0].shape[0]
precision_array[i] = float(tp_array[i]) / (
float(tp_array[i]) + float(fp_array[i]))
recall_array[i] = float(tp_array[i]) / (
float(tp_array[i]) + float(fn_array[i]))
stats_file.write('\n\t{} ({})\n'.format(project_name, id))
stats_file.write('\tNumber annotations : {}\n'.format(
project_data.shape[0]))
stats_file.write('\tTP : {} \tFN : {}\t| {}\n'.format(
tp_array[i], fn_array[i], (tp_array[i] + fn_array[i])))
stats_file.write('\tFP : {} \tTN : {}\t| {}\n'.format(
fp_array[i], tn_array[i], (fp_array[i] + tn_array[i])))
stats_file.write('\t -----------------------\n')
stats_file.write('\t {}\t\t{}\n'.format(
(tp_array[i] + fp_array[i]),
(fn_array[i] + tn_array[i])))
stats_file.write('\tPrecision : \t{}\n'.format(
precision_array[i]))
stats_file.write('\tRecall : \t{}\n'.format(
recall_array[i]))
# Plot : For each different model, scatter plot of all annotations of all projects
plt.scatter(pred_negative[:, 0],
pred_negative[:, 1],
c='r',
label="Actual negative",
alpha=0.6)
plt.scatter(pred_positive[:, 0],
pred_positive[:, 1],
c='g',
label="Actual positive",
alpha=0.7)
plt.title('Classification probability of annotations')
plt.xlabel('Annotations')
plt.ylabel('Probability (%)')
plt.ylim([-5, 105])
plt.xlim([-100, n_data + 100])
legend = plt.legend(loc='lower right', framealpha=0.1)
# Save plot figure
fig_name = model.strip('.pkl') + '.png'
fig_path = os.path.join(parameters['cytomine_working_path'],
"plots_colorspace0", fig_name)
if not os.path.exists(os.path.dirname(fig_path)):
os.makedirs(os.path.dirname(fig_path))
print fig_path
plt.savefig(fig_path)
plt.clf()
plt.close()
# Plot precision recall map for project : Each point is a project
plt.figure()
plt.scatter(recall_array * 100, precision_array * 100, c='b')
plt.title('Precision-Recall map')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.xlim([-5, 105])
plt.ylim([-5, 105])
plt.savefig(
os.path.join(parameters['cytomine_working_path'],
"plots_colorspace0",
os.path.dirname(fig_name),
"PR_" + os.path.basename(fig_name)))
plt.clf()
plt.close()
__author__ = "Vandaele Rémy <*****@*****.**>"
__contributors__ = ["Marée Raphaël <*****@*****.**>"]
__copyright__ = "Copyright 2010-2015 University of Liège, Belgium, http://www.cytomine.be/"
import cytomine
# connect to cytomine : parameters to set
cytomine_host = ""
cytomine_public_key = ""
cytomine_private_key = ""
id_project = 0
# Connection to Cytomine Core
conn = cytomine.Cytomine(cytomine_host,
cytomine_public_key,
cytomine_private_key,
base_path='/api/',
working_path='/tmp/',
verbose=True)
# define software parameter template
software = conn.add_software("Landmark_Model_Validation",
"pyxitSuggestedTermJobService",
"ValidateAnnotation")
conn.add_software_parameter("landmark_term", software.id, "Number", None, True,
1, False)
conn.add_software_parameter("landmark_r", software.id, "Number", None, True, 2,
False)
conn.add_software_parameter("landmark_rmax", software.id, "Number", None, True,
3, False)
conn.add_software_parameter("landmark_p", software.id, "Number", None, True, 4,
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key', type="string", default='XXX', dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key', type="string", default='YYY', dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path', type="string", default='/api/', dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_working_path', default="/tmp/", type="string", dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project', type="int", dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('--verbose', type="string", default="0", dest="verbose", help="Turn on (1) or off (0) verbose mode")
options, arguments = p.parse_args(args=sys.argv)
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['verbose'] = str2bool(options.verbose)
cytomine_connection = cytomine.Cytomine(parameters['cytomine_host'], parameters['cytomine_public_key'],
parameters['cytomine_private_key'],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=parameters['verbose'])
download_images(cytomine_connection, parameters['cytomine_id_project'])
download_annotations(cytomine_connection, parameters['cytomine_id_project'], parameters['cytomine_working_path'])
F.write('%d %d\n' % (xpos[(id_terms[i], image.id)],
ypos[(id_terms[i], image.id)]))
F.close()
return xpos, ypos
if __name__ == "__main__":
public_key = 'c3090afd-b793-4ae2-a5f3-2cfdaee01f32'
private_key = 'd7768538-db66-4aa6-882c-55f520f1be69'
wpath = sys.argv[1]
if (wpath.endswith('/')):
wpath = wpath + '/'
cytomine_connection = cytomine.Cytomine('demo.cytomine.be',
public_key,
private_key,
base_path='/api/',
working_path=wpath,
verbose=True)
droso_terms = [
6579647, 6581077, 6581992, 6583116, 6584107, 6585116, 6586002, 6587114,
6587962, 6588763, 6589668, 6590562, 6591526, 6592482, 6593390
]
cepha_terms = [
6625929, 6626956, 6628031, 6628982, 6630085, 6630930, 6632153, 6633169,
6634164, 6635158, 6636231, 6637186, 6638098, 6638869, 6639680, 6640638,
6641592, 6641602, 6641610
]
zebra_terms = [
6555577, 6555589, 6555603, 6555613, 6555621, 6555631, 6555631, 6555647,
6555657, 6555665, 6555675, 6555681, 6555691, 6555699, 6555709, 6555717,
def main(argv):
print("Main")
# Define command line options
p = optparse.OptionParser(description='Cytomine Segmentation prediction',
prog='Cytomine segmentation prediction',
version='0.1')
p.add_option(
'--cytomine_host',
type="string",
default='beta.cytomine.be',
dest="cytomine_host",
help="The Cytomine host (eg: beta.cytomine.be, localhost:8080)")
p.add_option('--cytomine_public_key',
type="string",
default='',
dest="cytomine_public_key",
help="Cytomine public key")
p.add_option('--cytomine_private_key',
type="string",
default='',
dest="cytomine_private_key",
help="Cytomine private key")
p.add_option('--cytomine_base_path',
type="string",
default='/api/',
dest="cytomine_base_path",
help="Cytomine base path")
p.add_option('--cytomine_id_software',
type="int",
dest="cytomine_id_software",
help="The Cytomine software identifier")
p.add_option('--cytomine_working_path',
default="/tmp/",
type="string",
dest="cytomine_working_path",
help="The working directory (eg: /tmp)")
p.add_option('--cytomine_id_project',
type="int",
dest="cytomine_id_project",
help="The Cytomine project identifier")
p.add_option('-i',
'--cytomine_id_image',
type='int',
dest='cytomine_id_image',
help="image id from cytomine",
metavar='IMAGE')
p.add_option('-z',
'--cytomine_zoom_level',
type='int',
dest='cytomine_zoom_level',
help="working zoom level")
p.add_option('-j',
'--nb_jobs',
type='int',
dest='nb_jobs',
help="number of parallel jobs")
p.add_option(
'--cytomine_predict_terms',
type='str',
dest='cytomine_predict_terms',
help=
"term id of all positive terms. The first term is the output predicted annotation term"
)
p.add_option('--cytomine_excluded_terms',
type='string',
dest='cytomine_excluded_terms',
help="term id of excluded terms)")
p.add_option('--pyxit_target_width',
type='int',
dest='pyxit_target_width',
help="pyxit subwindows width")
p.add_option('--pyxit_target_height',
type='int',
dest='pyxit_target_height',
help="pyxit subwindows height")
p.add_option('--pyxit_colorspace',
type='int',
dest='pyxit_colorspace',
help="pyxit colorspace encoding")
p.add_option('--pyxit_nb_jobs',
type='int',
dest='pyxit_nb_jobs',
help="pyxit number of jobs for trees")
p.add_option('--pyxit_fixed_size',
type='string',
default="0",
dest="pyxit_fixed_size",
help="extract fixed size subwindows")
p.add_option('--pyxit_transpose',
type='string',
default="0",
dest="pyxit_transpose",
help="transpose subwindows")
p.add_option('--pyxit_n_subwindows',
type='int',
default="10",
dest="pyxit_n_subwindows",
help="number of subwindows")
p.add_option('--pyxit_interpolation',
default=2,
type="int",
dest="pyxit_interpolation",
help="interpolation method 1,2,3,4")
p.add_option('--pyxit_min_size',
default=0.5,
type="float",
dest="pyxit_min_size",
help="min size")
p.add_option('--pyxit_max_size',
default=1.0,
type="float",
dest="pyxit_max_size",
help="max size")
p.add_option('--cytomine_reviewed',
type='string',
default="False",
dest="cytomine_reviewed",
help="Get reviewed annotations only")
p.add_option('--cytomine_dump_annotations',
type='string',
default="0",
dest="cytomine_dump_annotations",
help="Dump training annotations or not")
p.add_option('--cytomine_dump_annotation_stats',
type='string',
default="0",
dest="cytomine_dump_annotation_stats",
help="Calculate stats on dumped annotations or not")
p.add_option('--build_model',
type="string",
default="0",
dest="build_model",
help="Turn on (1) or off (0) model building")
p.add_option('--cytomine_annotation_projects',
type="string",
dest="cytomine_annotation_projects",
help="Projects from which annotations are extracted")
p.add_option('--verbose',
type="string",
default="0",
dest="verbose",
help="Turn on (1) or off (0) verbose mode")
p.add_option('--keras_save_to',
type='string',
default="",
dest='keras_save_to',
help="keras model weight file")
p.add_option('--keras_batch_size',
type="int",
dest="keras_batch_size",
help="Training batch size")
p.add_option('--keras_n_epochs',
type="int",
dest="keras_n_epochs",
help="Number of epochs")
p.add_option('--keras_shuffle',
type="string",
dest="keras_shuffle",
help="Turn on (1) or off (0) batch shuffle")
p.add_option('--keras_validation_split',
type="float",
dest="keras_validation_split",
help="Batch validation split")
options, arguments = p.parse_args(args=argv)
parameters = {}
parameters['keras_save_to'] = options.keras_save_to
parameters['keras_batch_size'] = options.keras_batch_size
parameters['keras_n_epochs'] = options.keras_n_epochs
parameters['keras_shuffle'] = options.keras_shuffle
parameters['keras_validation_split'] = options.keras_validation_split
parameters['cytomine_host'] = options.cytomine_host
parameters['cytomine_public_key'] = options.cytomine_public_key
parameters['cytomine_private_key'] = options.cytomine_private_key
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_working_path'] = options.cytomine_working_path
parameters['cytomine_base_path'] = options.cytomine_base_path
parameters['cytomine_id_project'] = options.cytomine_id_project
parameters['cytomine_id_software'] = options.cytomine_id_software
parameters['cytomine_predict_terms'] = map(
int, options.cytomine_predict_terms.split(','))
parameters['cytomine_predicted_annotation_term'] = parameters[
'cytomine_predict_terms'][0]
parameters['cytomine_excluded_terms'] = map(
int, options.cytomine_excluded_terms.split(','))
parameters['pyxit_colorspace'] = options.pyxit_colorspace
parameters['pyxit_nb_jobs'] = options.pyxit_nb_jobs
parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs
parameters['cytomine_nb_jobs'] = options.pyxit_nb_jobs
parameters['cytomine_id_image'] = options.cytomine_id_image
parameters['cytomine_zoom_level'] = options.cytomine_zoom_level
parameters['nb_jobs'] = options.nb_jobs
parameters['pyxit_target_width'] = options.pyxit_target_width
parameters['pyxit_target_height'] = options.pyxit_target_height
parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
parameters['pyxit_interpolation'] = options.pyxit_interpolation
parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
parameters['pyxit_min_size'] = options.pyxit_min_size
parameters['pyxit_max_size'] = options.pyxit_max_size
parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
parameters['cytomine_annotation_projects'] = map(
int, options.cytomine_annotation_projects.split(','))
parameters['cytomine_reviewed'] = str2bool(options.cytomine_reviewed)
parameters['cytomine_dump_annotation_stats'] = str2bool(
options.cytomine_dump_annotation_stats)
parameters['cytomine_dump_annotations'] = str2bool(
options.cytomine_dump_annotations)
parameters['build_model'] = str2bool(options.build_model)
parameters['dir_ls'] = os.path.join(
parameters["cytomine_working_path"],
str(parameters['cytomine_annotation_projects']).replace(
',', '-').replace('[', '').replace(']', '').replace(' ', ''),
"zoom_level", str(parameters['cytomine_zoom_level']))
pyxit_parameters = {}
pyxit_parameters['pyxit_target_width'] = options.pyxit_target_width
pyxit_parameters['pyxit_target_height'] = options.pyxit_target_height
pyxit_parameters['pyxit_n_subwindows'] = options.pyxit_n_subwindows
pyxit_parameters['pyxit_min_size'] = options.pyxit_min_size
pyxit_parameters['pyxit_max_size'] = options.pyxit_max_size
pyxit_parameters['pyxit_colorspace'] = options.pyxit_colorspace
pyxit_parameters['pyxit_interpolation'] = options.pyxit_interpolation
pyxit_parameters['pyxit_transpose'] = str2bool(options.pyxit_transpose)
pyxit_parameters['pyxit_fixed_size'] = str2bool(options.pyxit_fixed_size)
pyxit_parameters['pyxit_n_jobs'] = options.pyxit_nb_jobs
if options.verbose:
print(parameters)
# Create Cytomine connection
conn = cytomine.Cytomine(parameters["cytomine_host"],
parameters["cytomine_public_key"],
parameters["cytomine_private_key"],
base_path=parameters['cytomine_base_path'],
working_path=parameters['cytomine_working_path'],
verbose=str2bool(options.verbose))
# Dump annotations
if parameters['cytomine_dump_annotations']:
# Get annotation descriptions (JSON) from project(s)
annotations = None
for prj in parameters['cytomine_annotation_projects']:
if parameters["cytomine_reviewed"]:
annotations_prj = conn.get_annotations(
id_project=prj,
reviewed_only=parameters["cytomine_reviewed"])
else:
annotations_prj = conn.get_annotations(id_project=prj)
if not annotations:
annotations = annotations_prj
else:
annotations.data().extend(annotations_prj.data())
if prj == 21907448 or prj == 155194683:
annotations_prj = conn.get_annotations(id_project=prj,
id_term=91376951)
annotations.data().extend(annotations_prj.data())
print("Nb annotations so far... = %d" % len(annotations.data()))
print("Total annotations projects %s = %d" %
(parameters['cytomine_annotation_projects'],
len(annotations.data())))
# Set output dir parameters
if not os.path.exists(parameters['dir_ls']):
print("Creating annotation directory: %s" % parameters['dir_ls'])
os.makedirs(parameters['dir_ls'])
# Dump annotation images locally
print("Dump training annotation images in %s...", parameters['dir_ls'])
conn.dump_annotations(
annotations=annotations,
get_image_url_func=Annotation.get_annotation_alpha_crop_url,
dest_path=parameters['dir_ls'],
desired_zoom=parameters['cytomine_zoom_level'],
excluded_terms=parameters['cytomine_excluded_terms'])
# Put positive terms under the same term and same for negative terms
term_directories = os.listdir(parameters['dir_ls'])
pos_path = os.path.join(parameters['dir_ls'], "1")
if not os.path.exists(pos_path):
print("Creating positive annotation directory: %s" % pos_path)
os.makedirs(pos_path)
neg_path = os.path.join(parameters['dir_ls'], "0")
if not os.path.exists(neg_path):
print("Creating negative annotation directory: %s" % neg_path)
os.makedirs(neg_path)
for dir in term_directories:
dir_abs = os.path.join(parameters['dir_ls'], dir)
# Move files
if int(dir) in parameters['cytomine_predict_terms']:
for image_file in os.listdir(dir_abs):
os.rename(os.path.join(dir_abs, image_file),
os.path.join(pos_path, image_file))
else:
for image_file in os.listdir(dir_abs):
os.rename(os.path.join(dir_abs, image_file),
os.path.join(neg_path, image_file))
# Remove empty directory
if int(dir) != 0 and int(dir) != 1:
os.rmdir(dir_abs)
if parameters['cytomine_dump_annotation_stats']:
pos_path = os.path.join(parameters['dir_ls'], "1")
neg_path = os.path.join(parameters['dir_ls'], "0")
stats_dumped_annotations(pos_path, neg_path)
if parameters['build_model']:
print("Build_model...")
# Model name
model_name = "all_in_batchsize{}_epochs{}"\
.format(parameters['keras_batch_size'],
parameters['keras_n_epochs']).replace(".", "")
print("Model_name :", model_name)
pyxit = PyxitClassifier(
None,
n_subwindows=pyxit_parameters['pyxit_n_subwindows'],
min_size=pyxit_parameters['pyxit_min_size'],
max_size=pyxit_parameters['pyxit_max_size'],
target_width=pyxit_parameters['pyxit_target_width'],
target_height=pyxit_parameters['pyxit_target_height'],
n_jobs=pyxit_parameters['pyxit_n_jobs'],
interpolation=pyxit_parameters['pyxit_interpolation'],
transpose=pyxit_parameters['pyxit_transpose'],
colorspace=pyxit_parameters['pyxit_colorspace'],
fixed_size=pyxit_parameters['pyxit_fixed_size'],
random_state=None,
verbose=1,
get_output=_get_output_from_mask,
parallel_leaf_transform=False)
# Build filenames and classes
X, y = build_from_dir(parameters['dir_ls'])
classes = np.unique(y)
n_classes = len(classes)
y_original = y
y = np.searchsorted(classes, y)
n_images = len(y)
print("Number of images : ", n_images)
images, masks, labels = image_mask_builder(
X, y, parameters['pyxit_colorspace'])
# ImageDataGenerator : two instances with the same arguments
data_gen_args = dict(rotation_range=180.,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2,
rescale=1 / 255,
horizontal_flip=True,
vertical_flip=True)
# featurewise_center = True,
# featurewise_std_normalization = True)
image_datagen = ImageDataGenerator(**data_gen_args)
mask_datagen = ImageDataGenerator(**data_gen_args)
# Provide the same seed and keyword arguments to the fit and flow methods
seed = 1
# image_datagen.fit(images, augment = True, seed = seed)
# mask_datagen.fit(masks, augment = True, seed = seed)
print(type(images))
print(type(masks))
print(type(labels))
print(images[0:10])
print(masks[0:10])
print(labels[0:10])
image_generator = image_datagen.flow(images,
labels,
seed=seed,
shuffle=False)
mask_generator = mask_datagen.flow(masks,
labels,
seed=seed,
shuffle=False)
# combine generators into one which yields image and masks
train_generator = zip(image_generator, mask_generator)
# Creating and compiling model
if not os.path.exists(parameters['keras_save_to']):
os.makedirs(parameters['keras_save_to'])
model_weights_filename = os.path.join(parameters['keras_save_to'],
"weights_" + model_name + ".h5")
print('Fitting model...')
model = get_unet()
model_checkpoint = ModelCheckpoint(model_weights_filename,
monitor='val_loss',
save_best_only=True)
# Train FCN
model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=30,
callbacks=[model_checkpoint],
verbose=1)
