def main():
scanLungListPath = 'FileLists/ScansLungs.csv'
outfile = 'HistogramEdges_%s.txt' % datetime.datetime.now().timestamp()
prog = '../../Build/Statistics/DetermineHistogramBinEdges'
params = {
'scales' : ['1.2', '2.4', '4.8'],
'samples' : '10000',
'bins' : '49'
}
args = [
prog,
'--infile', scanLungListPath,
'--outfile', outfile,
'--samples', params['samples'],
'--bins', params['bins'],
] + list(intersperse('--scale', (s for s in params['scales'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error estimating histogram bin edges' )
return 1
return 0
def main():
skip = {
'Measure' : False,
}
basedir = ''
dirs = {
'Instances' : os.path.join(basedir, 'Data', 'Instances'),
'Statistics' : os.path.join(basedir, 'Data', 'Statistics', 'Scalability-1'),
'Bin' : '../../Build',
}
files = {
'Instances' : [
os.path.join(dirs['Instances'], 'instances500.csv'),
os.path.join(dirs['Instances'], 'instances1000.csv'),
os.path.join(dirs['Instances'], 'instances2500.csv'),
os.path.join(dirs['Instances'], 'instances5000.csv'),
os.path.join(dirs['Instances'], 'instances7500.csv'),
os.path.join(dirs['Instances'], 'instances10000.csv')
],
}
progs = {
'Scalability' : os.path.join(dirs['Bin'],'Experiments/02-ScalabilityOfClusteringAlgorithm/Scalability'),
}
params = {
'clusters' : [4, 8, 16, 32, 64],
'histograms' : 7*8, # 7 scales * 8 features
'burnin' : 10,
'iterations' : 100,
'branching' : 1,
}
if skip['Measure']:
print( 'Skipping: Measure' )
else:
print( 'Measuring' )
for instanceMatrix in files['Instances']:
args = [
progs['Scalability'],
'--input', instanceMatrix,
'--nHistograms', "%d" % params['histograms'],
'--output', os.path.join(dirs['Statistics'], 'stats_' + os.path.basename(instanceMatrix)),
'--burnin', "%d" % params['burnin'],
'--iterations', "%d" % params['iterations'],
'--branching', "%d" % params['branching']
] + list(intersperse('--clusters', ("%d" % k for k in params['clusters'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error measuring', instanceMatrix )
return 1
return 0
def main():
indir = '../Data/Dataset-3'
outdir = '../Data/Dataset-3/Training_10_10'
bindir = '../Build'
histDir = os.path.join(outdir, 'Histograms')
dirs = {
'Bags' : os.path.join(outdir, 'Bags'),
'FileLists' : os.path.join(outdir, 'FileLists'),
}
files = {
'ScanMaskList' : os.path.join(dirs['FileLists'], 'ScanLungList.txt'),
'HistogramSpec' : os.path.join(histDir, 'HistogramSpec.txt'),
}
progs = {
'MakeBag' : os.path.join(bindir,'FeatureExtraction/MakeBag'),
}
params = {
'scales' : ["%0.2f" % (0.6 * math.sqrt(2.0)**i) for i in [2,4,6]],
'num-rois' : 50,
'roi-size-x' : 41,
'roi-size-y' : 41,
'roi-size-z' : 41,
}
# Make the bags
print( 'Making bags' )
with open(files['ScanMaskList']) as infile:
scanMaskList = [(row[0].strip(), row[1].strip()) for row in csv.reader(infile)]
for scan, mask in scanMaskList:
print( "Using scan '%s'" % scan )
print( "Using mask '%s'" % mask )
args = [
progs['MakeBag'],
'--image', scan,
'--mask', mask,
'--histogram-spec', files['HistogramSpec'],
'--outdir', dirs['Bags'],
'--prefix', os.path.basename(scan),
'--num-rois', '%d' % params['num-rois'],
'--roi-size-x', '%d' % params['roi-size-x'],
'--roi-size-y', '%d' % params['roi-size-y'],
'--roi-size-z', '%d' % params['roi-size-z'],
] + list(intersperse('--scale', (s for s in params['scales'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error making bag' )
return 1
return 0
def main():
skip = {
'Make features' : False,
}
basedir = ''
dirs = {
'FileLists' : os.path.join(basedir, 'FileLists'),
'Features' : os.path.join(basedir, 'Data', 'Features'),
'Bin' : '../../Build',
}
files = {
'ImageMaskList' : os.path.join(dirs['FileLists'], 'ImageMaskList.csv'),
}
progs = {
'ExtractFeatures' : os.path.join(dirs['Bin'],'FeatureExtraction/ExtractFeatures'),
}
params = {
'scales' : ["0.6", "1.2"]
}
# Get the list of image/reference pairs
with open(files['ImageMaskList']) as infile:
imageMaskList = [
(row[0].strip(),
row[1].strip())
for row in csv.reader(infile)]
if skip['Make features']:
print( 'Skipping: Make features' )
else:
print( 'Making features' )
for image, mask in imageMaskList:
print( "Using image '%s'" % image )
print( "Using mask '%s'" % mask )
args = [
progs['ExtractFeatures'],
'--image', image,
'--mask', mask,
'--out', os.path.join(dirs['Features'], os.path.basename(mask))
] + list(intersperse('--scale', (s for s in params['scales'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error extracting features' )
return 1
return 0
def main():
scanLungListPath = 'FileLists/ScansLungs.csv'
outdir = 'Bags'
prog = '../../Build/FeatureExtraction/MakeBag'
params = {
'histogram-spec' : 'HistogramEdges_1456312478.268112.txt',
'roi-mask-value' : 6,
'scales' : ['1.2', '2.4', '4.8'],
'num-rois' : '50',
'roi-size-x' : '53',
'roi-size-y' : '53',
'roi-size-z' : '41',
}
# Make the bags
print( 'Making bags' )
with open(scanLungListPath) as infile:
scanLungList = [(row[0].strip(), row[1].strip())
for row in csv.reader(infile)]
for scan, lung in scanLungList:
print( "Using scan '%s'" % scan )
print( "Using lung mask '%s'" % lung )
args = [
prog,
'--image', scan,
'--mask', lung,
'--histogram-spec', params['histogram-spec'],
'--outdir', outdir,
'--prefix', os.path.basename(scan),
'--num-rois', params['num-rois'],
'--roi-size-x', params['roi-size-x'],
'--roi-size-y', params['roi-size-y'],
'--roi-size-z', params['roi-size-z'],
] + list(intersperse('--scale', (s for s in params['scales'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error making bag' )
return 1
return 0
def main():
indir = "../Data/Dataset-3"
outdir = "../Data/Dataset-3"
bindir = "../Build"
histDir = os.path.join(outdir, "Histograms")
dirs = {"Bags": os.path.join(outdir, "Bags")}
files = {
"ScanMaskList": os.path.join(indir, "ScanLungList.txt"),
"HistogramSpec": os.path.join(histDir, "HistogramSpec.txt"),
}
progs = {
"HistBinEdges": os.path.join(bindir, "Statistics/DetermineHistogramBinEdges"),
"MakeBag": os.path.join(bindir, "FeatureExtraction/MakeBag"),
}
params = {
"bins": 41,
"samples": 10000,
"scales": ["%0.2f" % (0.6 * math.sqrt(2.0) ** i) for i in range(7)],
"num-rois": 50,
"roi-size-x": 41,
"roi-size-y": 41,
"roi-size-z": 41,
}
# Determine the bin edges
print("Estimating bin edges for equalized histograms")
args = [
progs["HistBinEdges"],
"--infile",
files["ScanMaskList"],
"--outfile",
files["HistogramSpec"],
"--bins",
"%d" % params["bins"],
"--samples",
"%d" % params["samples"],
] + list(intersperse("--scale", (s for s in params["scales"])))
print(" ".join(args))
if subprocess.call(args) != 0:
print("Error running histogram bin edge estimation")
return 1
# Make the bags
print("Making bags")
with open(files["ScanMaskList"]) as infile:
scanMaskList = [(row[0].strip(), row[1].strip()) for row in csv.reader(infile)]
for scan, mask in scanMaskList:
print("Using scan '%s'" % scan)
print("Using mask '%s'" % mask)
args = [
progs["MakeBag"],
"--image",
scan,
"--mask",
mask,
"--histogram-spec",
files["HistogramSpec"],
"--outdir",
dirs["Bags"],
"--prefix",
os.path.basename(scan),
"--num-rois",
"%d" % params["num-rois"],
"--roi-size-x",
"%d" % params["roi-size-x"],
"--roi-size-y",
"%d" % params["roi-size-y"],
"--roi-size-z",
"%d" % params["roi-size-z"],
] + list(intersperse("--scale", (s for s in params["scales"])))
print(" ".join(args))
if subprocess.call(args) != 0:
print("Error making bag")
return 1
return 0
def main():
fileLists = {
'Scans' : 'FileLists/Scans.csv',
'Lungs' : 'FileLists/Lungs.csv',
'Regions' : 'FileLists/Regions.csv'
}
outdirs = {
'ROIs' : 'PatchSize21mm/ROIs',
'Bags' : 'PatchSize21mm/RegionBags'
}
progs = {
'MakeBag' : '../../Build/FeatureExtraction/MakeBag',
'GenerateROIs' : '../../Build/ROIExtraction/GenerateROIs',
}
params = {
'histogram-spec' : '../Training/HistogramEdges_bins25_1457110706.258915.txt',
'scales' : ['1.2', '2.4', '4.8'],
'num-rois' : '50',
'roi-size-x' : '27',
'roi-size-y' : '27',
'roi-size-z' : '21',
}
regions = [
('LowerLeft' , '1'),
('MiddleLeft' , '2'),
('UpperLeft' , '3'),
('LowerRight' , '4'),
('MiddleRight' , '5'),
('UpperRight' , '6'),
]
# First we generate the ROIs
print( 'Generating ROIs' )
with open(fileLists['Regions']) as infile:
regionList = [row[0].strip() for row in csv.reader(infile)]
roiList = []
for mask in regionList:
print( "Using region mask '%s'" % mask )
outfilename = os.path.basename( mask ) + '.ROIInfo'
roifiles = []
for region, value in regions:
outdir = os.path.join( outdirs['ROIs'], region )
if not os.path.isdir( outdir ):
try:
os.makedirs( outdir )
except OSError as e:
print( 'Could not create output directory:', outdir, e )
return 1
outpath = os.path.join( outdir, outfilename )
roifiles.append( outpath )
args = [
progs['GenerateROIs'],
'--mask', mask,
'--outfile', outpath,
'--num-rois', params['num-rois'],
'--roi-size-x', params['roi-size-x'],
'--roi-size-y', params['roi-size-y'],
'--roi-size-z', params['roi-size-z'],
'--mask-value', value
]
if subprocess.call( args ) != 0:
print( 'Error generating ROIs' )
return 1
# Concatenate the ROIs into a single big ROI file
outdir = os.path.join(outdirs['ROIs'], 'Combined' )
if not os.path.isdir( outdir ):
try:
os.makedirs( outdir )
except OSError as e:
print( 'Could not create output directory:', outdir, e )
return 1
outpath = os.path.join( outdir, outfilename )
roiList.append( outpath )
with open(outpath, 'w') as outfile:
with fileinput.input(files=roifiles) as infile:
outfile.writelines( infile )
# Now we extract the bags
with open(fileLists['Scans']) as infile:
scanList = [row[0].strip() for row in csv.reader(infile)]
with open(fileLists['Lungs']) as infile:
lungList = [row[0].strip() for row in csv.reader(infile)]
for scan, lung, roi in zip(scanList, lungList, roiList):
print( "Using scan '%s'" % scan )
print( "Using lung mask '%s'" % lung )
print( "Using ROI file '%s'" % roi )
args = [
progs['MakeBag'],
'--image', scan,
'--mask', lung,
'--histogram-spec', params['histogram-spec'],
'--outdir', outdirs['Bags'],
'--prefix', os.path.basename(scan),
'--roi-file', roi,
'--roi-file-has-header', '0',
] + list(intersperse('--scale', (s for s in params['scales'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error making bag' )
return 1
return 0
def main():
# Set to False to actually do something
skip = {
'Make bags' : False,
'Make instance matrices' : False,
}
basedir = './'
dirs = {
'Bags' : os.path.join(basedir, 'Bags'),
'Histograms' : os.path.join(basedir, 'Histograms'),
'FileLists' : os.path.join(basedir, 'FileLists'),
'Instances' : os.path.join(basedir, 'Instances'),
'Bin' : '../../Build'
}
files = {
'ScanMaskList' : os.path.join(dirs['FileLists'], 'ScanMaskList.csv'),
'HistogramSpec' : os.path.join(dirs['Histograms'], 'Dataset-3-HistogramSpec.txt'),
'InstanceMatrixBaseName' : os.path.join(dirs['Instances'], 'instances%d.csv'),
}
progs = {
'MakeBag' : os.path.join(dirs['Bin'],'FeatureExtraction/MakeBag'),
}
params = {
'scales' : ["%0.2f" % (0.6 * math.sqrt(2.0)**i) for i in range(7)],
'num-rois' : 50,
'roi-size-x' : 41,
'roi-size-y' : 41,
'roi-size-z' : 41,
}
# Make the bags
print( 'Making bags' )
with open(files['ScanMaskList']) as infile:
scanMaskList = [(row[0].strip(), row[1].strip()) for row in csv.reader(infile)]
if skip['Make bags']:
print( 'Skipping: Make bags' )
else:
print( 'Making bags' )
for scan, mask in scanMaskList:
print( "Using scan '%s'" % scan )
print( "Using mask '%s'" % mask )
args = [
progs['MakeBag'],
'--image', scan,
'--mask', mask,
'--histogram-spec', files['HistogramSpec'],
'--outdir', dirs['Bags'],
'--prefix', os.path.basename(scan),
'--num-rois', '%d' % params['num-rois'],
'--roi-size-x', '%d' % params['roi-size-x'],
'--roi-size-y', '%d' % params['roi-size-y'],
'--roi-size-z', '%d' % params['roi-size-z'],
] + list(intersperse('--scale', (s for s in params['scales'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error making bag' )
return 1
if skip['Make instance matrices']:
print( 'Skipping: Make instance matrices' )
else:
print( 'Making instance matrices' )
bags = [ os.path.join(dirs['Bags'],os.path.basename(scan) + 'bag.txt') for scan,_ in scanMaskList]
samples = []
for bag in bags:
with open(bag) as f:
samples += [row for row in csv.reader(f)]
outpath = files['InstanceMatrixBaseName'] % (params['num-rois'] * len(bags))
with open(outpath, 'w') as out:
csv.writer( out ).writerows( samples )
return 0
def main():
skip = {
'Measure' : False,
}
basedir = ''
dirs = {
'Instances' : os.path.join(basedir, 'Data', 'Instances'),
'Statistics' : os.path.join(basedir, 'Data', 'Statistics', 'Scalability-Thesis'),
'Bin' : '../../Build',
}
files = {
'Instances' : [
# os.path.join(dirs['Instances'], 'instances500.csv'),
# os.path.join(dirs['Instances'], 'instances1000.csv'),
# os.path.join(dirs['Instances'], 'instances2500.csv'),
# os.path.join(dirs['Instances'], 'instances5000.csv'),
#os.path.join(dirs['Instances'], 'instances7500.csv'),
os.path.join(dirs['Instances'], 'instances10000.csv')
],
}
progs = {
'Scalability' : os.path.join(dirs['Bin'],'Experiments/02-ScalabilityOfClusteringAlgorithm/Scalability'),
}
params = {
'histograms' : 7*8, # 7 scales * 8 features
'burnin' : 10,
'iterations' : 100,
'Scalability-Thesis-1': {
'clusters' : [2, 4, 8, 16, 32, 64],
'branching' : 2,
'kmeans-iterations' : 11,
},
'Scalability-Thesis-2': {
'clusters' : [2, 4, 8, 16, 32, 64],
'branching' : 1,
'kmeans-iterations' : 11,
},
'Scalability-Thesis-3': {
'clusters' : [2, 4, 8, 16, 32, 64],
'branching' : 2,
'kmeans-iterations' : 100,
},
'Scalability-Thesis-4': {
'clusters' : [2, 4, 8, 16, 32, 64],
'branching' : 1,
'kmeans-iterations' : 100,
},
}
experiments = [
# 'Scalability-Thesis-1',
# 'Scalability-Thesis-2',
# 'Scalability-Thesis-3',
'Scalability-Thesis-4',
]
for ex in experiments:
if ex in skip:
print( 'Skipping:', ex )
else:
print( 'Running', ex )
for instanceMatrix in files['Instances']:
outname = '%s_%s' % (ex, os.path.basename(instanceMatrix))
args = [
progs['Scalability'],
'--input', instanceMatrix,
'--nHistograms', "%d" % params['histograms'],
'--output', os.path.join(dirs['Statistics'], outname),
'--burnin', "%d" % params['burnin'],
'--iterations', "%d" % params['iterations'],
'--branching', "%d" % params[ex]['branching'],
'--kmeans-iterations', "%d" % params[ex]['kmeans-iterations'],
] + list(intersperse('--clusters', ("%d" % k for k in params[ex]['clusters'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error measuring', instanceMatrix )
return 1
return 0
def main(argv=None):
if argv is None:
argv = sys.argv
if len(argv) < 5:
print( "Usage:", argv[0],
"<image-mask-list> <outdir> <tmpdir> <scale>+" )
return 1
progs = {
'MaskedNormalizedConvolution' : '../Build/FeatureExtraction/MaskedNormalizedConvolution',
'MaskedImageFilter' : '../Build/FeatureExtraction/MaskedImageFilter',
'GradientFeatures' : '../Build/FeatureExtraction/FiniteDifference_GradientFeatures',
'HessianFeatures' : '../Build/FeatureExtraction/FiniteDifference_HessianFeatures',
}
imageMaskListPath = argv[1]
outDir = argv[2]
tmpDir = argv[3]
try:
scales = [ float(x) for x in argv[4:] ]
except ValueError as e:
print( "<scale> arguments must be numbers" )
return 1
try:
with open(imageMaskListPath) as f:
imageMaskList = [(row[0], row[1]) for row in csv.reader(f)]
except FileNotFoundError as e:
print( "<image-mask-list> is not a valid path" )
return 1
except IndexError as e:
print( "<image-mask-list> must contain two columns" )
return 2
scaleMatcher = re.compile('.*(scale_[0-9]*\.[0-9]*)')
for image, mask in imageMaskList:
# Run MaskedNormalizedConvolution
baseName = os.path.basename( image )[:-4]
print( baseName )
args = [
progs['MaskedNormalizedConvolution'],
"--image", image,
"--certainty", mask,
"--outdir", tmpDir,
"--prefix", "unmasked" + baseName + "_",
"--maskoutput", "0",
] + list(intersperse("--scale", ("%0.2f" % s for s in scales)))
if subprocess.call(args) != 0:
print( 'Failed to process:', image )
continue
# Now we extract features from the unmasked images
# We want to name the images according to scale
paths = glob.glob( os.path.join( tmpDir, 'unmasked*' ) )
for path in paths:
scale = scaleMatcher.match(path).group(1)
prefix = baseName + "_" + scale + "_"
print( scale )
# Intensity features
args = [
progs['MaskedImageFilter'],
"--image", path,
"--mask", mask,
"--outdir", outDir,
"--prefix", prefix,
]
print( ' '.join(args) )
if subprocess.call( args ) != 0:
print( 'Failed to extract gradient features' )
continue
# Gradient features
args = [
progs['GradientFeatures'],
"--image", path,
"--mask", mask,
"--outdir", outDir,
"--prefix", prefix,
]
print( ' '.join(args) )
if subprocess.call( args ) != 0:
print( 'Failed to extract gradient features' )
continue
# Hessian features
args = [
progs['HessianFeatures'],
"--image", path,
"--mask", mask,
"--outdir", outDir,
"--prefix", prefix,
]
print( ' '.join(args) )
if subprocess.call( args ) != 0:
print( 'Failed to extract Hessian features' )
continue
# Delete the unmasked image
os.remove( path )
def main():
skip = {
'Stability-1' : False,
'Stability-2' : False,
'Stability-3' : False,
'Stability-4' : False,
'Stability-5' : False,
}
basedir = ''
dirs = {
'Instances' : os.path.join(basedir, 'Data', 'Instances'),
'Distances' : os.path.join(basedir, 'Distances'),
'Bin' : '../../Build',
}
files = {
'Instances' : [
os.path.join(dirs['Instances'], 'instances1000.csv'),
],
}
progs = {
'Stability' : os.path.join(dirs['Bin'],'Experiments/03-StabilityOfClusteringAlgorithm/Stability'),
}
params = {
'histograms' : 7*8, # 7 scales * 8 features
'iterations' : 100,
'Stability-1' : {
'clusters' : [2, 4, 8, 16, 32, 64],
'branching' : 2,
},
'Stability-2' : {
'clusters' : [4, 7, 16, 31, 64],
'branching' : 4,
},
'Stability-3' : {
'clusters' : [8, 15, 29, 64],
'branching' : 8,
},
'Stability-4' : {
'clusters' : [16, 31, 61],
'branching' : 16,
},
'Stability-5' : {
'clusters' : [32, 63],
'branching' : 32,
},
}
experiments = [
'Stability-1',
'Stability-2',
'Stability-3',
'Stability-4',
'Stability-5'
]
for ex in experiments:
if skip[ex]:
print( 'Skipping:', ex )
else:
print( 'Running:', ex )
for instanceMatrix in files['Instances']:
outname = '%s_%s' % (ex,
os.path.basename(instanceMatrix))
args = [
progs['Stability'],
'--input', instanceMatrix,
'--histograms', "%d" % params['histograms'],
'--output', os.path.join(dirs['Distances'], outname),
'--iterations', "%d" % params['iterations'],
'--branching', "%d" % params[ex]['branching']
] + list(intersperse('--clusters', ("%d" % k for k in params[ex]['clusters'])))
print(' '.join(args))
if subprocess.call( args ) != 0:
print( 'Error running', ex, '.Using:', instanceMatrix )
return 1
return 0
