def test_sima(self):
"""
(BlockMethod) with SIMA strategy
"""
# NOTE: this test was brittle and failed non-deterministically with any
# more than one source
import sima.segment
# construct the SIMA strategy
simaStrategy = sima.segment.STICA(components=1)
simaStrategy.append(sima.segment.SparseROIsFromMasks(min_size=20))
simaStrategy.append(sima.segment.SmoothROIBoundaries())
simaStrategy.append(sima.segment.MergeOverlapping(threshold=0.5))
tsc = ThunderContext(self.sc)
data = tsc.makeExample('sources',
dims=(60, 60),
centers=[[20, 15]],
noise=0.5,
seed=42)
# create and fit the thunder extraction strategy
strategy = SourceExtraction('sima', simaStrategy=simaStrategy)
model = strategy.fit(data, size=(30, 30))
assert (model.count == 1)
# check that the one center is recovered
ep = 1.5
assert (model[0].distance([20, 15]) < ep)
def test_local_max(self):
"""
(FeatureMethod) localmax with defaults
"""
tsc = ThunderContext(self.sc)
data = tsc.makeExample('sources', dims=[60, 60], centers=[[10, 10], [40, 40]], noise=0.0, seed=42)
model = SourceExtraction('localmax').fit(data)
# order is irrelevant, but one of these must be true
cond1 = (model[0].distance([10, 10]) == 0) and (model[1].distance([40, 40]) == 0)
cond2 = (model[0].distance([40, 40]) == 0) and (model[1].distance([10, 10]) == 0)
assert(cond1 or cond2)
def test_nmf(self):
"""
(BlockMethod) nmf with defaults
"""
tsc = ThunderContext(self.sc)
data = tsc.makeExample('sources', dims=(60, 60), centers=[[20, 20], [40, 40]], noise=0.1, seed=42)
model = SourceExtraction('nmf', componentsPerBlock=1).fit(data, size=(30, 30))
# order is irrelevant, but one of these must be true
ep = 0.50
cond1 = (model[0].distance([20, 20]) < ep) and (model[1].distance([40, 40]) < ep)
cond2 = (model[0].distance([40, 40]) < ep) and (model[1].distance([20, 20]) < ep)
assert(cond1 or cond2)
def test_sima(self):
"""
(BlockMethod) with SIMA strategy
"""
import sima.segment
# construct the SIMA strategy
simaStrategy = sima.segment.STICA(components=2)
simaStrategy.append(sima.segment.SparseROIsFromMasks(min_size=20))
simaStrategy.append(sima.segment.SmoothROIBoundaries())
simaStrategy.append(sima.segment.MergeOverlapping(threshold=0.5))
tsc = ThunderContext(self.sc)
data = tsc.makeExample('sources', dims=(60, 60), centers=[[20, 15], [40, 45]], noise=0.1, seed=42)
# create and fit the thunder extraction strategy
strategy = SourceExtraction('sima', simaStrategy=simaStrategy)
model = strategy.fit(data, size=(30, 30))
# order is irrelevant, but one of these must be true
ep = 1.5
cond1 = (model[0].distance([20, 15]) < ep) and (model[1].distance([40, 45]) < ep)
cond2 = (model[1].distance([20, 15]) < ep) and (model[0].distance([40, 45]) < ep)
assert(cond1 or cond2)
def setUp(self):
super(TestContextLoading, self).setUp()
self.tsc = ThunderContext(self.sc)
def setUp(self):
super(TestLoadIrregularImages, self).setUp()
self.tsc = ThunderContext(self.sc)
def CompressImages(inputs, output, confObj):
debugMode = False
st = datetime.now()
imageExt = confObj['ext']
imageHeight = confObj['dims'][0]
imageWidth = confObj['dims'][1]
refImgId = confObj['refImageId']
diffImageFolder = confObj['DiffImageFolder']
if debugMode == True:
print confObj
import glob
totImages = len(glob.glob(inputs + "*." + imageExt))
if os.path.exists(output):
shutil.rmtree(output)
conf = SparkConf().setAppName('ImgCompress')
sc = SparkContext(conf=conf)
imageHeight = sc.broadcast(imageHeight)
imageWidth = sc.broadcast(imageWidth)
tsc = ThunderContext(sc)
tscImages = tsc.loadImages(inputs, (imageHeight.value, imageWidth.value),
imageExt, imageExt).cache()
floatingPixelRdd = tscImages.rdd.flatMapValues(lambda r: r).zipWithIndex().map(lambda l: ((l[0][0],(l[1]-(l[0][0]*int(imageHeight.value)))),l[0][1]))\
.flatMapValues(lambda r: r).zipWithIndex().\
map(lambda l: ((l[0][0][1],(l[1]-(l[0][0][0]*int(imageWidth.value)*int(imageHeight.value) + l[0][0][1]*int(imageWidth.value)))),(l[0][0][0],l[0][1])))
if debugMode == True:
floatingPixelRdd.saveAsTextFile(output + "\\Stage-1-FloatingPixel")
temporalVoxelRdd = floatingPixelRdd.groupByKey().map(
lambda x: (x[0], list(x[1]))).cache()
if debugMode == True:
temporalVoxelRdd.saveAsTextFile(output + "\\Stage-2-TemporalVoxel")
iMapImageDict = {}
for imgIndex in range(totImages):
imgIndexBD = sc.broadcast(imgIndex)
#-------------------------------HELPER FUNCTIONS-------------------------------------
def ReAdjustImapList(l):
intList = l[1]
mKey = intList.pop(imgIndexBD.value)[1]
return (mKey, intList)
def calculateMedainForEachImage(l):
intRdd = sc.parallelize(l[1]).flatMap(
lambda l: l).groupByKey().map(lambda x: (x[0], list(x[1])))
intRddSorted = intRdd.map(lambda l: (l[0], sorted(l[1])))
intRddMedian = intRddSorted.map(lambda l:
(l[0], l[1][int(len(l[1]) / 2)]))
return intRddMedian.collect()
#-------------------------------HELPER FUNCTIONS-------------------------------------
imapRdd = temporalVoxelRdd.map(lambda l: ReAdjustImapList(l)).sortBy(lambda l: l[0], False)\
.groupByKey().map(lambda x : (x[0], list(x[1])))
if debugMode == True:
imapRdd.saveAsTextFile(output + "\\Stage-3__IMAP-Stage-1_imgIdx-" +
str(imgIndex))
imapRdd = imapRdd.flatMapValues(lambda l: l).flatMapValues(lambda l: l).map(lambda l: ((l[0],l[1][0]),l[1][1])).\
groupByKey().map(lambda x : (x[0], list(x[1]))).\
map(lambda l: (l[0],sorted(l[1]))).map(lambda l: (l[0], l[1][int(len(l[1])/2)])).\
map(lambda l: (l[0][0],(l[0][1], l[1])))
if debugMode == True:
imapRdd.saveAsTextFile(output + "\\Stage-3__IMAP-Stage-2_imgIdx-" +
str(imgIndex))
imapRdd = imapRdd.groupByKey().map(
lambda x: (x[0], sorted(list(x[1]), key=lambda k: k[0])))
if debugMode == True:
imapRdd.saveAsTextFile(output + "\\Stage-3__IMAP-Stage-3_imgIdx-" +
str(imgIndex))
imapDict = imapRdd.collectAsMap()
iMapImageDict[imgIndex] = imapDict
iMapDictBD = sc.broadcast(iMapImageDict)
refImageIdBD = sc.broadcast(refImgId)
def CalcMapValue(pix, iMapVal):
pImgIdx = pix[0]
residual = 0
for iIdx in iMapVal:
if pImgIdx == iIdx[0]:
residual = int(iIdx[1]) - pix[1]
break
return (pix[0], residual)
def ApplyIMap(l):
voxel = l[1]
refIntensity = l[1][refImageIdBD.value][1]
iMapValues = iMapDictBD.value[refImageIdBD.value][refIntensity]
resdualValues = []
for pixel in voxel:
if pixel[0] != refImageIdBD.value:
resdualValues.append(CalcMapValue(pixel, iMapValues))
else:
resdualValues.append(pixel)
return (l[0], resdualValues)
diffFVRdd = temporalVoxelRdd.map(lambda l: ApplyIMap(l))
if debugMode == True:
diffFVRdd.saveAsTextFile(output + "\\Stage-4__Diff-Stage-1")
residualImages = diffFVRdd.flatMapValues(lambda l: l).map(lambda l: ((l[1][0], l[0][0]),(l[0][1],l[1][1])))\
.groupByKey().map(lambda x : (x[0], sorted(list(x[1]),key = lambda k: (k[0]))))\
.map(lambda l: (l[0][0],(l[0][1],l[1])))\
.groupByKey().map(lambda x : (x[0], sorted(list(x[1]),key = lambda k: (k[0]))))\
.map(lambda l: (l[0], removeRCIndex(l[1])))
residualImages.saveAsTextFile(output + "\\" + diffImageFolder)
for ingIdx in range(totImages):
iMapImageDict[str(ingIdx)] = dict([
(str(k), str(v)) for k, v in iMapImageDict[ingIdx].items()
])
del iMapImageDict[ingIdx]
import json
with open(output + "\\imap.json", 'w') as f:
json.dump(iMapImageDict, f)
with open(output + '\\conf.json', 'w') as f:
json.dump(confObj, f)
# clean up
sc.stop()
en = datetime.now()
td = en - st
print td
