def make_embed(project_name):
proj = db.session.query(Project).filter_by(name=project_name).first()
if proj is None:
return jsonify(error=f"project {project_name} doesn't exist"), 400
model0ExistOrNot = os.path.exists(
f"./projects/{project_name}/models/0/best_model.pth")
current_app.logger.info(
f'Model 0 (autoencoder) exists = {model0ExistOrNot}')
if not model0ExistOrNot:
return jsonify(
error=
"Embedding is not available unless at least a base model is trained. Please make patches and train AE"
), 400
if proj.train_ae_time is None and proj.iteration == 0:
error_message = f'The base model 0 of project {project_name} was overwritten when Retrain Model 0 started.\n ' \
f'Please wait until the Retrain Model 0 finishes. '
current_app.logger.warn(error_message)
return jsonify(error=error_message), 400
current_app.logger.info('Checking if the embeddings are the most recent.')
# get config options:
batchsize = config.getint('make_embed', 'batchsize', fallback=32)
patchsize = config.getint('make_embed', 'patchsize', fallback=256)
numimgs = request.args.get('numimgs', default=-1, type=int)
modelid = request.args.get('modelid',
default=get_latest_modelid(project_name),
type=int)
outdir = f"./projects/{project_name}/models/{modelid}"
latest_modelID = get_latest_modelid(project_name)
if modelid < 0 or modelid > latest_modelID:
return jsonify(
error=
f"Your selected Embed Model ID is {modelid}. The last model ID is {latest_modelID}. A valid Model ID ranges from 0 to {latest_modelID}."
), 400
# get the command:
full_command = [
sys.executable, "make_embed.py", project_name, f"-o{outdir}",
f"-p{patchsize}", f"-b{batchsize}", f"-m{numimgs}"
]
current_app.logger.info(f'Full command = {str(full_command)}')
# update the embedding iteration:
# current_app.logger.info('Updating the embedding iteration to the model iteration:')
# proj.embed_iteration = proj.iteration
db.session.commit()
# run the command asynchronously:
command_name = "make_embed"
return pool_run_script(project_name,
command_name,
full_command,
callback=make_embed_callback)
def get_prediction(project_name, image_name):
current_app.logger.info(
f'Getting prediction for project {project_name} and image {image_name}'
)
project = Project.query.filter_by(name=project_name).first()
curr_image = Image.query.filter_by(projId=project.id,
name=image_name).first()
if curr_image is None:
jsonify(error=f"Image {image_name} does not exist"), 400
modelid = request.args.get('model',
get_latest_modelid(project_name),
type=int)
current_app.logger.info(f'Model id = {str(modelid)}')
if modelid <= 0:
current_app.logger.warn(
f"No DL model trained for {project_name} -- {image_name} -- {modelid}"
)
return jsonify(
error="No AI model trained, so no AI results available yet."), 400
upload_folder = f"./projects/{project_name}/pred/{modelid}"
fname = image_name.replace(".png", "_pred.png")
full_fname = f"{upload_folder}/{fname}"
current_app.logger.info('Full filename for prediction = ' + full_fname)
print('Generating new prediction image:')
batchsize = config.getint('get_prediction', 'batchsize', fallback=32)
patchsize = config.getint('get_prediction', 'patchsize', fallback=256)
# run the command:
full_command = [
sys.executable, "make_output_unet_cmd.py", f"-s{batchsize}",
f"-p{patchsize}",
f"-m./projects/{project_name}/models/{modelid}/best_model.pth",
f"-o./projects/{project_name}/pred/{modelid}",
f"./projects/{project_name}/{image_name}", "--force"
]
command_name = "generate_prediction"
return pool_get_image(project_name,
command_name,
full_command,
full_fname,
imageid=curr_image.id)
def get_torch_device(gpuid=None):
# Output a torch device with a preferred gpuid (use -2 to force cpu)
if not gpuid:
gpuid = config.getint("cuda", "gpuid", fallback=0)
device = torch.device(
gpuid if gpuid != -2 and torch.cuda.is_available() else 'cpu')
return device
def rendered_project_image(template_name, project_name, image_name):
project = Project.query.filter_by(name=project_name).first()
image = Image.query.filter_by(projId=project.id, name=image_name).first()
defaultCropSize = config.getint('common', 'patchsize', fallback=256)
return render_template(template_name,
project=project,
image=image,
defaultCropSize=defaultCropSize)
def train_autoencoder(project_name):
proj = db.session.query(Project).filter_by(name=project_name).first()
if proj is None:
return jsonify(error=f"project {project_name} doesn't exist"), 400
current_app.logger.info(
f'Training autoencoder for project {project_name}:')
# get the config options:
current_app.logger.info(f'Getting config options:')
num_images = config.getint('train_ae', 'numimages', fallback=-1)
batch_size = config.getint('train_ae', 'batchsize', fallback=32)
patch_size = config.getint('train_ae', 'patchsize', fallback=256)
num_workers = config.getint('train_ae', 'numworkers', fallback=0)
num_epochs = config.getint('train_ae', 'numepochs', fallback=1000)
num_epochs_earlystop = config.getint('train_ae',
'num_epochs_earlystop',
fallback=-1)
num_min_epochs = config.getint('train_ae', 'num_min_epochs', fallback=300)
current_app.logger.info(
f'Images = {num_images}, epochs = {num_epochs}, batch size = {batch_size}'
)
# get the command:
full_command = [
sys.executable, "train_ae.py", f"-n{num_epochs}", f"-p{patch_size}",
f"-s{num_epochs_earlystop}", f"-l{num_min_epochs}", f"-m{num_images}",
f"-b{batch_size}", f"-r{num_workers}",
f"-o./projects/{project_name}/models/0",
f"./projects/{project_name}/patches/*.png"
]
current_app.logger.info(full_command)
# run it asynchronously:
command_name = "train_autoencoder"
# Set proj.train_ae_time = null since the model 0 is being retrained, the time should be unavailable
proj.train_ae_time = None
db.session.commit()
return pool_run_script(project_name,
command_name,
full_command,
callback=train_autoencoder_callback)
def annotation(project_name, image_name):
project = Project.query.filter_by(name=project_name).first()
if not project:
return render_template("error.html")
# Method 1
# image = Image.query.filter_by(projId=project.id, name=image_name).first()
# image.nROIs = Roi.query.filter_by(imageId=image.id).count()
# image.nTrainingROIs = Roi.query.filter_by(imageId=image.id, testingROI=0).count()
# Method 2 (corresponding sql code)
# SELECT image.id, count(roi.id)
# FROM image
# JOIN roi
# ON roi.imageId = image.id
# WHERE image.id = 1
# GROUP BY image.id
image = db.session.query(Image.id, Image.projId, Image.name, Image.path, Image.height, Image.width, Image.date,
Image.rois, Image.make_patches_time, Image.nobjects,
db.func.count(Roi.id).label('nROIs'),
(db.func.count(Roi.id) - db.func.ifnull(db.func.sum(Roi.testingROI), 0))
.label('nTrainingROIs')). \
outerjoin(Roi, Roi.imageId == Image.id). \
filter(Image.name == image_name).filter(Image.projId == project.id).group_by(Image.id).first()
x = request.args.get('startX', "#")
y = request.args.get('startY', "#")
defaultCropSize = config.getint('common', 'patchsize', fallback=256)
return render_template("annotation.html",
project=project,
image=image,
startX=x,
startY=y,
defaultCropSize=defaultCropSize)
import os
import logging
from sqlalchemy.ext.automap import automap_base
from flask import jsonify, current_app, send_from_directory
from flask_sqlalchemy import SQLAlchemy
from QA_worker import run_script
import QA_db
from QA_db import Job, JobidBase, set_job_status
from QA_config import config
# seconds to sleep before re-querying the server:
retry_seconds = config.getint('frontend', 'retry_seconds', fallback=20)
db = SQLAlchemy()
jobs_logger = logging.getLogger('jobs')
################################################################################
# Check if a job with this (command & parameters) is running or queued.
#
# We output that job. If the job does not exist or is in a "DONE"/"ERROR"
# state, we return None.
def getUnfinishedJob(project_name, command, imageid):
# get the job:
project_id = QA_db.get_project_id(project_name)
current_app.logger.info(f'Checking if a job is free for project {project_id} ("{project_name}").')
job = Job.query.filter_by(
def get_superpixels(project_name, image_name):
current_app.logger.info(
f'Getting superpixel for project {project_name} and image {image_name}'
)
latest_modelid = get_latest_modelid(project_name)
force = request.args.get('force', False, type=bool)
modelidreq = request.args.get('superpixel_run_id',
latest_modelid,
type=int)
current_app.logger.info(f'Model id = {str(modelidreq)}')
if modelidreq > latest_modelid:
return jsonify(
error=
f"Requested ModelID {modelidreq} greater than available models {latest_modelid}"
), 400
project = Project.query.filter_by(name=project_name).first()
curr_image = Image.query.filter_by(projId=project.id,
name=image_name).first()
superpixel_modelid = curr_image.superpixel_modelid
current_app.logger.info(
f'The current superpixel_modelid of {image_name} = {str(superpixel_modelid)}'
)
upload_folder = f"./projects/{project_name}/superpixels"
spixel_fname = image_name.replace(".png", "_superpixels.png")
full_fname = f"{upload_folder}/{spixel_fname}"
current_app.logger.info('Full filename for superpixel = ' + full_fname)
batchsize = config.getint('superpixel', 'batchsize', fallback=32)
patchsize = config.getint('superpixel', 'patchsize', fallback=256)
approxcellsize = config.getint('superpixel', 'approxcellsize', fallback=20)
compactness = config.getfloat('superpixel', 'compactness', fallback=.01)
command_to_use = config.get("superpixel",
'command_to_use',
fallback="make_superpixel.py")
if modelidreq < 0:
# We are using simple method, since we have no dl model
current_app.logger.warn(
f"No DL model trained for {project_name} -- {image_name} -- {modelidreq}, will use simple method"
)
command_to_use = "make_superpixel.py"
full_command = [
sys.executable, command_to_use, f"-p{patchsize}", f"-x{batchsize}",
f"-c{compactness}", f"-a{approxcellsize}",
f"-m./projects/{project_name}/models/{modelidreq}/best_model.pth",
f"-s./projects/{project_name}/superpixels/",
f"-o./projects/{project_name}/superpixels_boundary/",
f"./projects/{project_name}/{image_name}", "--force"
]
current_app.logger.info(
f'We are running {command_to_use} to generate superpixels for IMAGE {image_name} in PROJECT {project_name} '
)
current_app.logger.info(f'Superpixel command = {full_command}')
command_name = "generate_superpixel"
if modelidreq > superpixel_modelid or force:
try:
os.remove(full_fname)
except:
pass
return pool_get_image(project_name,
command_name,
full_command,
full_fname,
imageid=curr_image.id,
callback=get_superpixels_callback)
def make_patches(project_name):
# pull this project from the database:
current_app.logger.info(
f'Getting project info from database for project {project_name}.')
project = db.session.query(Project).filter_by(name=project_name).first()
if project is None:
current_app.logger.warn(
f'Unable to find {project_name} in database. Returning HTML response code 400.'
)
return jsonify(error=f"Project {project_name} does not exist"), 400
target_files = []
current_app.logger.info('Looping through images.')
for img in project.images:
current_app.logger.info(
f'Checking database if patches have been computed for image "{img.name}".'
)
needs_calculating = False
if img.make_patches_time:
current_app.logger.info(
'Database claims that the patches have been computed. Checking filesystem.'
)
image_name_without_extension = os.path.splitext(
img.name)[0] # <-- remove extension
current_app.logger.info(f'Image {image_name_without_extension}')
patches_pattern = f'./projects/{project_name}/patches/{image_name_without_extension}*.png'
current_app.logger.info(f'Patches pattern = {patches_pattern}')
number_of_patches = len(glob.glob(patches_pattern))
current_app.logger.info(f'Number of patches = {number_of_patches}')
if number_of_patches == 0:
current_app.logger.warn(
'The database is incorrectly reporting that patches exist. We are recomputing them since no patches exist on the filesystem for this image.'
)
needs_calculating = True
else:
needs_calculating = True
if needs_calculating:
current_app.logger.info(
f'Patches need to be computed for image at {img.path}. Adding this image to the list.'
)
target_files.append(img.path)
# img.patches_computed = True # note, this only goes through when commit is called
current_app.logger.info(
'Marked patches_computed to be True in the database.')
if not target_files:
error_message = 'No pending target image files for making patches.'
current_app.logger.warn(error_message)
return jsonify(error=error_message), 400
current_app.logger.info(
'Storing image filenames for patches in text file:')
with open(f"./projects/{project_name}/patches/new_imgs.txt",
"w") as textfile:
for fname in target_files:
textfile.write(f"{fname}\n")
patchsize = config.getint('make_patches', 'patchsize', fallback=256)
# get the command:
full_command = [
sys.executable, "make_patches_for_embed.py", f"-p{patchsize}",
f"-o./projects/{project_name}/patches/",
f"./projects/{project_name}/patches/new_imgs.txt"
]
whiteBG = request.args.get("whiteBG", default="keep", type=str)
if whiteBG == "remove":
full_command.append("-b")
current_app.logger.info(full_command)
# close the db session and note that patches_computed is true:
db.session.commit()
# run the command asynchronously
command_name = "make_patches"
return pool_run_script(project_name,
command_name,
full_command,
callback=make_patches_callback)
def retrain_dl(project_name):
proj = Project.query.filter_by(name=project_name).first()
if proj is None:
return jsonify(error=f"project {project_name} doesn't exist"), 400
current_app.logger.info(
f'About to train a new transfer model for {project_name}')
frommodelid = request.args.get('frommodelid', default=0, type=int)
if (frommodelid == -1):
frommodelid = get_latest_modelid(project_name)
if frommodelid > proj.iteration or not os.path.exists(
f"./projects/{project_name}/models/{frommodelid}/best_model.pth"):
return jsonify(
error=f"Deep learning model {frommodelid} doesn't exist"), 400
if proj.train_ae_time is None and frommodelid == 0:
error_message = f'The base model 0 of project {project_name} was overwritten when Retrain Model 0 started.\n ' \
f'Please wait until the Retrain Model 0 finishes. '
current_app.logger.warn(error_message)
return jsonify(error=error_message), 400
# todo: make sure there's actually a model in that subdirectory since errors still create the dir before the model is ready
new_modelid = get_latest_modelid(project_name) + 1
output_model_path = f"./projects/{project_name}/models/{new_modelid}/"
current_app.logger.info(f'New model path = {output_model_path}')
# store the list of test and training images in text files:
test_file_path = f"projects/{project_name}/test_imgs.txt"
train_file_path = f"projects/{project_name}/train_imgs.txt"
current_app.logger.info('Populating project files:')
populate_training_files(project_name, train_file_path, test_file_path)
# check if enough data exists:
empty_training = not os.path.exists(test_file_path) or os.stat(
test_file_path).st_size == 0
empty_testing = not os.path.exists(test_file_path) or os.stat(
test_file_path).st_size == 0
if empty_training or empty_testing: # TODO can improve this by simply counting ROIs in the db
error_message = f'Not enough training/test images for project {project_name}. You need at least 1 of each.'
current_app.logger.warn(error_message)
return jsonify(error=error_message), 400
# get config properties:
num_epochs = config.getint('train_tl', 'numepochs', fallback=1000)
num_epochs_earlystop = config.getint('train_tl',
'num_epochs_earlystop',
fallback=-1)
num_min_epochs = config.getint('train_tl', 'num_min_epochs', fallback=300)
batch_size = config.getint('train_tl', 'batchsize', fallback=32)
patch_size = config.getint('train_tl', 'patchsize', fallback=256)
num_workers = config.getint('train_tl', 'numworkers', fallback=0)
edge_weight = config.getfloat('train_tl', 'edgeweight', fallback=2)
pclass_weight = config.getfloat('train_tl', 'pclass_weight', fallback=.5)
fillbatch = config.getboolean('train_tl', 'fillbatch', fallback=False)
# query P/N pixel count from database for ppixel_train npixel_train ppixel_test npixel_test
if pclass_weight == -1:
proj_ppixel = db.session.query(db.func.sum(
Image.ppixel)).filter_by(projId=proj.id).scalar()
proj_npixel = db.session.query(db.func.sum(
Image.npixel)).filter_by(projId=proj.id).scalar()
total = proj_npixel + proj_ppixel
pclass_weight = 1 - proj_ppixel / total
# get the command to retrain the model:
full_command = [
sys.executable, "train_model.py", f"-p{patch_size}",
f"-e{edge_weight}", f"-n{num_epochs}", f"-s{num_epochs_earlystop}",
f"-l{num_min_epochs}", f"-b{batch_size}", f"-o{output_model_path}",
f"-w{pclass_weight}", f"-r{num_workers}",
f"-m./projects/{project_name}/models/{frommodelid}/best_model.pth",
f"./projects/{project_name}"
]
if (fillbatch):
full_command.append("--fillbatch")
current_app.logger.info(f'Training command = {full_command}')
# run the script asynchronously:
command_name = "retrain_dl"
return pool_run_script(project_name,
command_name,
full_command,
callback=retrain_dl_callback)
