def createPatches(name, dataDir, patchListFile):
"""Create patch files from images in dataDir.
Input:
- $dataDir/images/*.png
- $dataDir/expert/*_expert.png
Output:
- $dataDir/patches/{0,1}/*_$i_$j.png
- $dataDir/patches/$patchListFile: List of patch files and patch index
"""
printSectionHeader('Creating %s patches' % name)
imageFiles = glob.glob(os.path.join(dataDir, "images", "*.png"))
patchesDir = os.path.join(dataDir, "patches")
for i in range(2):
utils.ensureDirectoryExists(os.path.join(patchesDir, str(i)))
with open(os.path.join(patchesDir, patchListFile), "w") as patchListFile:
for i, imageFile in enumerate(imageFiles):
print('\nCreating patches for %s file %d/%d - %s' %
(name, i + 1, len(imageFiles), imageFile))
imageName = os.path.basename(os.path.splitext(imageFile)[0])
extractPatchesFromImage(dataDir, imageName, patchesDir,
patchListFile)
def splitControlTumorData():
"""Split the data created from the images in the directories Controls
and LargeTumor in input_data_root via splitData and put the
results in training and testing subdirectories of
output_data_root.
"""
# Input data directories
controlInputDir = os.path.join(input_data_root, "Controls")
tumorInputDir = os.path.join(input_data_root, "LargeTumor")
# Output data directories
controlOutputDir = os.path.join(output_data_root, "controls")
tumorOutputDir = os.path.join(output_data_root, "tumors")
trainOutputDir = os.path.join(output_data_root, "training")
testOutputDir = os.path.join(output_data_root, "testing")
# Sanity checks
utils.ensureDirectoryExists(trainOutputDir)
utils.ensureDirectoryExists(testOutputDir)
# Process control files
splitData('control', controlInputDir, controlOutputDir, trainOutputDir,
testOutputDir)
# Process tumor files
splitData('tumor', tumorInputDir, tumorOutputDir, trainOutputDir,
testOutputDir)
def __init__(self, pdftoolbox_path, cache_path, activation_code, tmp_folder, webroot_folder):
self.pdftoolbox_path = pdftoolbox_path
self.cache_path = cache_path
self.logger = logging.getLogger('printflow2')
self.activation_code = activation_code
self.ready = True
utils.ensureDirectoryExists(tmp_folder)
self.tmp_folder = tmp_folder
self.webroot_folder = webroot_folder
def run():
# set output directory
outputDir = os.path.join(output_data_root, "testing", "cnn")
utils.ensureDirectoryExists(outputDir)
sys.stdout = utils.Logger(os.path.join(outputDir, 'net_test.log'))
# Model definition
modelDef = os.path.join(
output_data_root, "NetProto", "net_best.hdf5")
if os.path.isfile(modelDef):
print("Model definition " + modelDef + " found.")
else:
print('Error : model definition "' + modelDef + '" not found.\n' +
' Train neural network before processing it.')
sys.exit(0)
# Create network
net = M.load_model(modelDef)
# get list of test mha files
testMhaFiles = glob.glob(os.path.join(
testDataDir, "images", "*_zslab.mha"))
# segment all test .mha files
for mhaFile in testMhaFiles:
testAnimal = os.path.basename(os.path.splitext(mhaFile)[0])[:-6]
print "Segmenting ", testAnimal + ".mha", " ..."
# segment all slabs
for i in range(script_params['NUM_SLABS']):
inputFilename = os.path.join(
testDataDir, "images", str(i) + "_" + testAnimal + '_zslab.png')
outputFilename = os.path.join(
outputDir, str(i) + "_" + testAnimal + '_zslab.png')
segmentSlab(net, inputFilename, outputFilename)
# reconstruct volume from segmented slabs by mapping back to their MIP
# location
reconstructSegVolume(testAnimal, outputDir)
# segment tubes using ridge traversal
vascularModelFile = os.path.join(input_data_root, 'vascularModel.mtp')
segmentTubes(testAnimal, vascularModelFile, outputDir,
script_params['VESSEL_SEED_PROBABILITY'],
script_params['VESSEL_SCALE'])
break
def run_job(self, serial_no, file_path, file_name, profile, success_destinationation, fail_destination, guid):
status = -1
if self.ready:
try:
theFile = os.path.join(file_path, file_name)
report_folder = os.path.join(os.path.join(self.webroot_folder, guid.replace('-', '/')))
utils.ensureDirectoryExists(report_folder)
report_file_mask = os.path.join(report_folder, 'report_'+file_name+'.pdf')
if os.path.exists(report_file_mask):
try:
os.remove(report_file_mask)
except:
self.logger.error('Problem removing tmp pdf report file: '+report_file_mask)
report_file_text = os.path.join(report_folder, 'report_'+file_name+'.txt')
if os.path.exists(report_file_text):
try:
os.remove(report_file_text)
except:
self.logger.error('Problem removing tmp text report file: '+report_file_mask)
report_file_html = os.path.join(report_folder, 'report_'+file_name+'.html')
if os.path.exists(report_file_html):
try:
os.remove(report_file_html)
except:
self.logger.error('Problem removing tmp html report file: '+report_file_mask)
self.logger.info('Processing file:' + theFile)
#license = self.get_license(serial_no)
#print "License:", license
processResult = None
cmd = ['"'+str(self.pdftoolbox_path)+'"', '--license='+str(serial_no), '--secret='+str(self.get_license(serial_no)), \
'--nosummary', '--noprogress', '--nohits', '--nofixups', \
'--report=MASK,ALWAYS,OVERVIEW,PATH="'+str(report_file_mask)+'"', \
'--report=XSLT=compacttext_point,ALWAYS,PATH="'+str(report_file_text)+'"', \
'--report=XSLT=compacthtml_point,ALWAYS,PATH="'+str(report_file_html)+'"', \
'"'+str(profile)+'"', '"'+str(theFile)+'"']
cmd_str = ' '.join(cmd)
p = subprocess.Popen(cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
processResult = p.communicate()
if p.returncode is not None:
status = p.returncode
except subprocess.CalledProcessError:
# check return status to see what pdftoolbox thinks
self.logger.error('Problem calling command: '+str(cmd))
traceback.print_exc()
else:
self.logger.error('pdftoolbox not correctly initialised')
return status, processResult, report_folder, report_file_mask, report_file_text, report_file_html
def createDB(name, patchesDir, patchListFile, dbDir):
"""Create a database in dbDir from the patches in patchesDir, indexed
by patchListFile
"""
print('Creating %s DB ...\n' % name)
if os.path.exists(dbDir):
shutil.rmtree(dbDir)
utils.ensureDirectoryExists(dbDir)
patchListFile = os.path.join(patchesDir, patchListFile)
db = utils.open_sqlite3_db(dbDir)
db.execute('''create table "Patches" (
"filename" text,
"patch_index" integer,
-- Interpreted as a square C-major-order uint8 array with each
-- dimension (2 * PATCH_RADIUS + 1)
"image_data" blob
)''')
lines = list(open(patchListFile))
random.shuffle(lines)
def generate_data():
for i, l in enumerate(lines):
filename, patch_index = l.rstrip().rsplit(' ', 1)
patch_index = int(patch_index)
image_data = buffer(
skimage.io.imread(os.path.join(patchesDir, filename)).copy())
yield filename, patch_index, image_data
if i % 1000 == 0:
print("{} patches processed".format(i))
db.executemany('''insert into "Patches" values (?, ?, ?)''',
generate_data())
db.commit()
db.close()
def run_preview(self, serial_no, file_path, file_name, guid):
status = -1
if self.ready:
theFile = os.path.join(file_path, file_name)
preview_folder = os.path.join(self.webroot_folder, guid.replace('-', '/'))
utils.ensureDirectoryExists(preview_folder)
preview_file_name = file_name+'.jpg'
previewFile = os.path.join(preview_folder, preview_file_name)
try:
cmd = ['"'+str(self.pdftoolbox_path)+'"', '--license='+str(serial_no), '--secret='+str(self.get_license(serial_no)), \
'--saveasimg', '--noprogress', '--pagerange=1', '--imgformat=JPEG', '--resolution=400x400', '--compression=JPEG_high',\
'--outputfile="'+str(previewFile)+'"', \
'"'+str(theFile)+'"']
cmd_str = ' '.join(cmd)
p = subprocess.Popen(cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
processResult = p.communicate()
except subprocess.CalledProcessError:
# check return status to see what pdftoolbox thinks
self.logger.error('Problem calling command: '+str(cmd))
traceback.print_exc()
return status, processResult, previewFile, preview_file_name
def createZMIPSlabs(name, inputDir, outputDir):
"""Process all image files in immediate subdirectories of inputDir to
correspondingly prefixed images in outputDir. outputDir is
created if it doesn't already exist. The subdirectory structure
is not replicated.
See the documentation of createZMIPSlabsForFile for the exact
files created.
"""
# Sanity check
utils.ensureDirectoryExists(outputDir)
# Process files
printSectionHeader('Creating Z-MIP slabs for %ss' % name)
mhdFiles = glob.glob(os.path.join(inputDir, "*", "*.mhd"))
for i, mhdFile in enumerate(mhdFiles):
print("\n%s file %d/%d : %s" % (name, i + 1, len(mhdFiles), mhdFile))
createZMIPSlabsForFile(mhdFile, outputDir)
import keras
import keras.callbacks as C
import keras.layers as L
import keras.models as M
import keras.optimizers as O
import keras.regularizers as R
import keras.utils as U
# Define file paths
net_proto_path = os.path.join(output_data_root, 'NetProto')
snapshot_format = os.path.join(net_proto_path, 'net').replace('{','{{').replace('}','}}') + \
'_{epoch:02d}.hdf5'
train_results_dir = os.path.join(net_proto_path, 'train_results')
utils.ensureDirectoryExists(train_results_dir)
patch_size = 2 * script_params['PATCH_RADIUS'] + 1
# Features square plot :
# Plot any layer's output features with the data parameter corresponding to
# solver.net.params['conv1'][0].data, where 'conv1' is the layer name.
def squarePlot(data):
"""Take an array of shape (n, height, width) or (n, height, width, 3)
and visualize each (height, width) thing in a grid of size approx. sqrt(n) by sqrt(n)"""
# normalize data for display
data = (data - data.min()) / (data.max() - data.min())
def extractPatchesFromImage(rootDir, imageName, outputDir, patchListFile):
"""Convert an image to a set of patches. Patches are sorted into
"positive" and "negative" patches, i.e. those with and without a
vessel at the center. Positive patches have index 1, negative
patches index 0. Patch relative paths and their indices are
written to the patchListFile file object.
Input:
- $rootDir/images/$imageName.png: The image to extract slices from
- $rootDir/expert/$imageName_expert.png: The corresponding expert mask
Output:
- $outputDir/0/$imageName/$i_$j.png: Negative patches
- $outputDir/1/$imageName/$i_$j.png: Positive patches
"""
def writePatch(patchSetIndex, i, j):
"""Write out a patch centered at i,j to a correspondingly named file,
using the given patch set index. Create a corresponding entry
in patchListFile.
"""
psi = str(patchSetIndex)
filename = os.path.join(psi, imageName, str(i) + "_" + str(j) + ".png")
patchListFile.write(filename + " " + psi + "\n")
skimage.io.imsave(os.path.join(outputDir, filename),
inputImage[i - w:i + w + 1, j - w:j + w + 1])
for i in range(2):
utils.ensureDirectoryExists(os.path.join(outputDir, str(i), imageName))
# patch/window radius
w = script_params['PATCH_RADIUS']
total_pos_patches = script_params[
'POSITIVE_PATCHES_PER_INPUT_FILE'] / script_params['NUM_SLABS']
total_neg_patches = script_params[
'NEGATIVE_TO_POSITIVE_RATIO'] * total_pos_patches
num_patch_types = 4
vessel_ctl_pos, other_pos, vessel_bnd_neg, other_neg = range(
num_patch_types)
patch_index = np.array(
dict_to_list({
vessel_ctl_pos: 1,
other_pos: 1,
vessel_bnd_neg: 0,
other_neg: 0,
}))
frac = np.array(
dict_to_list({
vessel_ctl_pos:
script_params['POSITIVES_NEAR_VESSEL_CENTERLINE'],
other_pos:
script_params['OTHER_POSITIVES'],
vessel_bnd_neg:
script_params['NEGATIVES_NEAR_VESSEL_BOUNDARY'],
other_neg:
script_params['OTHER_NEGATIVES'],
}))
for i in range(2):
if abs(frac[patch_index == i].sum() - 1.0) > 1e-9:
raise ValueError("{} patch fraction sum must be 1.0, is {}".format(
"Positive" if i else "Negative", patch_frac_sum))
# read input image
inputImageFile = os.path.join(rootDir, "images", imageName + '.png')
inputImage = skimage.io.imread(inputImageFile)
# read expert segmented label mask
expertSegFile = os.path.join(rootDir, "expert", imageName + '_expert.png')
trainingMaskFile = os.path.join(rootDir, "expert",
imageName + '_train.png')
computeTrainingMask(expertSegFile, trainingMaskFile)
expertSeg = skimage.io.imread(expertSegFile)
trainingMask = skimage.io.imread(trainingMaskFile)
# Iterate through expert mask and find pos/neg patches
# Slice that we want, which excludes edge pixels
s = np.s_[w:-w - 1]
s = (s, s)
trainingMaskMid = trainingMask[s]
expertSegMid = expertSeg[s]
vessel_ctl_pos_mask = trainingMaskMid > 0.6 * 255
mask = dict_to_list({
# Vessel centerline pixel (positive)
vessel_ctl_pos:
vessel_ctl_pos_mask,
# Other vessel pixel (positive)
other_pos: (expertSegMid > 0) & ~vessel_ctl_pos_mask,
# Vessel bound pixel (negative)
vessel_bnd_neg: (trainingMaskMid > 0) & ~vessel_ctl_pos_mask,
# Other background pixel (negative)
other_neg: (expertSegMid == 0) & (trainingMaskMid == 0),
})
indices = [np.array(np.where(m)).T + w for m in mask]
# Desired number of each type of patch (not necessarily a whole number)
desiredFractionalPatches = frac * np.where(patch_index, total_pos_patches,
total_neg_patches)
availablePatches = np.array([len(ind) for ind in indices])
availableFraction = availablePatches.astype(
float) / desiredFractionalPatches
# The largest fraction we can take, capped at 1
minAvailableFraction = min(availableFraction.min(), 1.)
if minAvailableFraction != 1.:
print(
"WARNING: Too few of a desired patch type; " +
"scaling all types down by {}% accordingly".format(
minAvailableFraction * 100))
numPatches = np.ceil(desiredFractionalPatches * minAvailableFraction -
1e-9).astype(int)
for key, label in enumerate(patch_index):
selInd = random.sample(indices[key], numPatches[key])
for i, j in selInd:
writePatch(label, i, j)
