def MatchingIteration(p):
# compute gradient for matching
(grad_m, VEnergy, IEnergy) = MatchingGradient(p)
if p.optMethod == 'FIXEDGD':
# take fixed stepsize gradient step
ca.Add_MulC_I(p.m0, grad_m, -p.stepSize)
else:
raise Exception("Unknown optimization scheme: " + p.optMethod)
# end if
return (VEnergy, IEnergy)
def RunWarpIteration(subid, cf, p, t, Imsmts, cpinds, cpstates, msmtinds,
gradAtMsmts, it):
# compute gradients
(grad_m, p.sumJac, p.sumSplatI, VEnergy,
IEnergy) = WarpGradient(p, t, Imsmts, cpinds, cpstates, msmtinds,
gradAtMsmts)
# TODO: do energy related stuff for printing and bookkeeping
# WarpPlots()
if cf.optim.method == 'FIXEDGD':
# take gradient descent step
ca.Add_MulC_I(p.m0, grad_m, -p.stepSize)
else:
raise Exception("Unknown optimization scheme: " + cf.optim.optMethod)
# end if
return VEnergy, IEnergy
def BuildHGM(cf):
"""Worker for running Hierarchical Geodesic Model (HGM)
n for group geodesic estimation on a subset of individuals.
Runs HGM on this subset sequentially. The variations retuned
are summed up to get update for all individuals"""
size = Compute.GetMPIInfo()['size']
rank = Compute.GetMPIInfo()['rank']
name = Compute.GetMPIInfo()['name']
localRank = Compute.GetMPIInfo()['local_rank']
nodename = socket.gethostname()
# prepare output directory
common.Mkdir_p(os.path.dirname(cf.io.outputPrefix))
# just one reporter process on each node
isReporter = rank == 0
cf.study.numSubjects = len(cf.study.subjectIntercepts)
if isReporter:
# Output loaded config
if cf.io.outputPrefix is not None:
cfstr = Config.ConfigToYAML(HGMConfigSpec, cf)
with open(cf.io.outputPrefix + "parsedconfig.yaml", "w") as f:
f.write(cfstr)
#common.DebugHere()
# if MPI check if processes are greater than number of subjects. it is okay if there are more subjects than processes
if cf.compute.useMPI and (cf.study.numSubjects < cf.compute.numProcesses):
raise Exception("Please don't use more processes " +
"than total number of individuals")
# subdivide data, create subsets for this thread to work on
nodeSubjectIds = cf.study.subjectIds[rank::cf.compute.numProcesses]
nodeIntercepts = cf.study.subjectIntercepts[rank::cf.compute.numProcesses]
nodeSlopes = cf.study.subjectSlopes[rank::cf.compute.numProcesses]
nodeBaselineTimes = cf.study.subjectBaselineTimes[rank::cf.compute.
numProcesses]
sys.stdout.write(
"This is process %d of %d with name: %s on machinename: %s and local rank: %d.\nnodeIntercepts: %s\n nodeSlopes: %s\n nodeBaselineTimes: %s\n"
% (rank, size, name, nodename, localRank, nodeIntercepts, nodeSlopes,
nodeBaselineTimes))
# mem type is determined by whether or not we're using CUDA
mType = ca.MEM_DEVICE if cf.compute.useCUDA else ca.MEM_HOST
# load data in memory
# load intercepts
J = [
common.LoadITKImage(f, mType) if isinstance(f, str) else f
for f in nodeIntercepts
]
# load slopes
n = [
common.LoadITKField(f, mType) if isinstance(f, str) else f
for f in nodeSlopes
]
# get imGrid from data
imGrid = J[0].grid()
# create time array with checkpointing info for group geodesic
(t, Jind, gCpinds) = HGMSetUpTimeArray(cf.optim.nTimeStepsGroup,
nodeBaselineTimes, 0.0000001)
tdiscGroup = CAvmHGMCommon.HGMSetupTimeDiscretizationGroup(
t, J, n, Jind, gCpinds, mType, nodeSubjectIds)
# create time array with checkpointing info for residual geodesic
(s, scratchInd, rCpinds) = HGMSetUpTimeArray(cf.optim.nTimeStepsResidual,
[1.0], 0.0000001)
tdiscResidual = CAvmHGMCommon.HGMSetupTimeDiscretizationResidual(
s, rCpinds, imGrid, mType)
# create group state and residual state
groupState = CAvmHGMCommon.HGMGroupState(
imGrid,
mType,
cf.vectormomentum.diffOpParamsGroup[0],
cf.vectormomentum.diffOpParamsGroup[1],
cf.vectormomentum.diffOpParamsGroup[2],
t,
cf.optim.NIterForInverse,
cf.vectormomentum.varIntercept,
cf.vectormomentum.varSlope,
cf.vectormomentum.varInterceptReg,
cf.optim.stepSizeGroup,
integMethod=cf.optim.integMethodGroup)
#ca.Copy(groupState.I0, common.LoadITKImage('/usr/sci/projects/ADNI/nikhil/software/vectormomentumtest/TestData/FlowerData/Longitudinal/GroupGeodesic/I0.mhd', mType))
# note that residual state is treated a scratch variable in this algorithm and reused for computing residual geodesics of multiple individual
residualState = CAvmHGMCommon.HGMResidualState(
None,
None,
imGrid,
mType,
cf.vectormomentum.diffOpParamsResidual[0],
cf.vectormomentum.diffOpParamsResidual[1],
cf.vectormomentum.diffOpParamsResidual[2],
s,
cf.optim.NIterForInverse,
cf.vectormomentum.varIntercept,
cf.vectormomentum.varSlope,
cf.vectormomentum.varInterceptReg,
cf.optim.stepSizeResidual,
integMethod=cf.optim.integMethodResidual)
# start up the memory manager for scratch variables
ca.ThreadMemoryManager.init(imGrid, mType, 0)
# need some host memory in np array format for MPI reductions
if cf.compute.useMPI:
mpiImageBuff = None if mType == ca.MEM_HOST else ca.Image3D(
imGrid, ca.MEM_HOST)
mpiFieldBuff = None if mType == ca.MEM_HOST else ca.Field3D(
imGrid, ca.MEM_HOST)
for i in range(len(groupState.t) - 1, -1, -1):
if tdiscGroup[i].J is not None:
indx_last_individual = i
break
'''
# initial template image
ca.SetMem(groupState.I0, 0.0)
tmp = ca.ManagedImage3D(imGrid, mType)
for tdisc in tdiscGroup:
if tdisc.J is not None:
ca.Copy(tmp, tdisc.J)
groupState.I0 += tmp
del tmp
if cf.compute.useMPI:
Compute.Reduce(groupState.I0, mpiImageBuff)
# divide by total num subjects
groupState.I0 /= cf.study.numSubjects
'''
# run the loop
for it in range(cf.optim.Niter):
# compute HGM variation for group
HGMGroupVariation(groupState, tdiscGroup, residualState, tdiscResidual,
cf.io.outputPrefix, rank, it)
common.CheckCUDAError("Error after HGM iteration")
# compute gradient for momenta (m is used as scratch)
# if there are multiple nodes we'll need to sum across processes now
if cf.compute.useMPI:
# do an MPI sum
Compute.Reduce(groupState.sumSplatI, mpiImageBuff)
Compute.Reduce(groupState.sumJac, mpiImageBuff)
Compute.Reduce(groupState.madj, mpiFieldBuff)
# also sum up energies of other nodes
# intercept
Eintercept = np.array([groupState.EnergyHistory[-1][1]])
mpi4py.MPI.COMM_WORLD.Allreduce(mpi4py.MPI.IN_PLACE,
Eintercept,
op=mpi4py.MPI.SUM)
groupState.EnergyHistory[-1][1] = Eintercept[0]
Eslope = np.array([groupState.EnergyHistory[-1][2]])
mpi4py.MPI.COMM_WORLD.Allreduce(mpi4py.MPI.IN_PLACE,
Eslope,
op=mpi4py.MPI.SUM)
groupState.EnergyHistory[-1][2] = Eslope[0]
ca.Copy(groupState.m, groupState.m0)
groupState.diffOp.applyInverseOperator(groupState.m)
ca.Sub_I(groupState.m, groupState.madj)
#groupState.diffOp.applyOperator(groupState.m)
# now take gradient step in momenta for group
if cf.optim.method == 'FIXEDGD':
# take fixed stepsize gradient step
ca.Add_MulC_I(groupState.m0, groupState.m, -cf.optim.stepSizeGroup)
else:
raise Exception("Unknown optimization scheme: " + cf.optim.method)
# end if
# now divide to get the new base image for group
ca.Div(groupState.I0, groupState.sumSplatI, groupState.sumJac)
# keep track of energy in this iteration
if isReporter and cf.io.plotEvery > 0 and ((
(it + 1) % cf.io.plotEvery == 0) or (it == cf.optim.Niter - 1)):
HGMPlots(cf,
groupState,
tdiscGroup,
residualState,
tdiscResidual,
indx_last_individual,
writeOutput=True)
if isReporter:
(VEnergy, IEnergy, SEnergy) = groupState.EnergyHistory[-1]
print datetime.datetime.now().time(
), " Iter", it, "of", cf.optim.Niter, ":", VEnergy + IEnergy + SEnergy, '(Total) = ', VEnergy, '(Vector) + ', IEnergy, '(Intercept) + ', SEnergy, '(Slope)'
# write output images and fields
HGMWriteOutput(cf, groupState, tdiscGroup, isReporter)
def ApplyAffine(Iout, Im, A, bg=ca.BACKGROUND_STRATEGY_PARTIAL_ZERO):
'''Applies an Affine matrix A to an image Im using the Image3D
grid (size, spacing, origin) of the two images (Input and Output)
'''
# algorithm outline: Create a temporary large grid, then perform
# real affine transforms here, then crop to be the size of the out grid
ca.SetMem(Iout, 0.0)
A = np.matrix(A)
bigsize = [max(Iout.grid().size().x, Im.grid().size().x),
max(Iout.grid().size().y, Im.grid().size().y),
max(Iout.grid().size().z, Im.grid().size().z)]
idgrid = ca.GridInfo(ca.Vec3Di(bigsize[0], bigsize[1], bigsize[2]),
ca.Vec3Df(1, 1, 1),
ca.Vec3Df(0, 0, 0))
# newgrid = Iout.grid() # not a true copy!!!!!
newgrid = ca.GridInfo(Iout.grid().size(),
Iout.grid().spacing(),
Iout.grid().origin())
mType = Iout.memType()
Imbig = cc.PadImage(Im, bigsize)
h = ca.Field3D(idgrid, mType)
ca.SetToIdentity(h)
if isinstance(Im, ca.Field3D):
Ioutbig = ca.Field3D(idgrid, mType)
else:
Ioutbig = ca.Image3D(idgrid, mType)
# note: x_real' = A*x_real; x_real' given (input grid)
# solution: x_real = A^-1 * x_real
# where x_real = x_index*spacing + origin
# and x_real' = x_index'*spacing' + origin'
# x_index' is really given, as is both spacings/origins
# and we plug in the solution for x_index' into applyH
if A.shape[1] == 3: # 2D affine matrix
x = ca.Image3D(idgrid, mType)
y = ca.Image3D(idgrid, mType)
xnew = ca.Image3D(idgrid, mType)
ynew = ca.Image3D(idgrid, mType)
ca.Copy(x, h, 0)
ca.Copy(y, h, 1)
# convert x,y to world coordinates
x *= Iout.grid().spacing().x
y *= Iout.grid().spacing().y
x += Iout.grid().origin().x
y += Iout.grid().origin().y
# Matrix Multiply (All in real coords)
Ainv = A.I
ca.MulC_Add_MulC(xnew, x, Ainv[0, 0], y, Ainv[0, 1])
ca.MulC_Add_MulC(ynew, x, Ainv[1, 0], y, Ainv[1, 1])
xnew += (Ainv[0, 2])
ynew += (Ainv[1, 2]) # xnew and ynew are now in real coords
# convert back to index coordinates
xnew -= Im.grid().origin().x
ynew -= Im.grid().origin().y
xnew /= Im.grid().spacing().x
ynew /= Im.grid().spacing().y
ca.SetToZero(h)
ca.Copy(h, xnew, 0)
ca.Copy(h, ynew, 1)
elif A.shape[1] == 4: # 3D affine matrix
x = ca.Image3D(idgrid, mType)
y = ca.Image3D(idgrid, mType)
z = ca.Image3D(idgrid, mType)
xnew = ca.Image3D(idgrid, mType)
ynew = ca.Image3D(idgrid, mType)
znew = ca.Image3D(idgrid, mType)
ca.Copy(x, h, 0)
ca.Copy(y, h, 1)
ca.Copy(z, h, 2)
x *= Iout.grid().spacing().x
y *= Iout.grid().spacing().y
z *= Iout.grid().spacing().z
x += Iout.grid().origin().x
y += Iout.grid().origin().y
z += Iout.grid().origin().z
# Matrix Multiply (All in real coords)
Ainv = A.I
ca.MulC_Add_MulC(xnew, x, Ainv[0, 0], y, Ainv[0, 1])
ca.Add_MulC_I(xnew, z, Ainv[0, 2])
xnew += (Ainv[0, 3])
ca.MulC_Add_MulC(ynew, x, Ainv[1, 0], y, Ainv[1, 1])
ca.Add_MulC_I(ynew, z, Ainv[1, 2])
ynew += (Ainv[1, 3])
ca.MulC_Add_MulC(znew, x, Ainv[2, 0], y, Ainv[2, 1])
ca.Add_MulC_I(znew, z, Ainv[2, 2])
znew += (Ainv[2, 3])
# convert to index coordinates
xnew -= Im.grid().origin().x
ynew -= Im.grid().origin().y
znew -= Im.grid().origin().z
xnew /= Im.grid().spacing().x
ynew /= Im.grid().spacing().y
znew /= Im.grid().spacing().z
ca.Copy(h, xnew, 0)
ca.Copy(h, ynew, 1)
ca.Copy(h, znew, 2)
Imbig.setGrid(idgrid)
ca.ApplyH(Ioutbig, Imbig, h, bg)
# crop Ioutbig -> Iout
ca.SubVol(Iout, Ioutbig, ca.Vec3Di(0, 0, 0))
Iout.setGrid(newgrid) # change back
def GeoRegIteration(subid, cf, p, t, Imsmts, cpinds, cpstates, msmtinds,
gradAtMsmts, EnergyHistory, it):
# compute gradient for regression
(grad_m, sumJac, sumSplatI, VEnergy,
IEnergy) = GeoRegGradient(p, t, Imsmts, cpinds, cpstates, msmtinds,
gradAtMsmts)
# do energy related stuff for printing and bookkeeping
#if it>0:
EnergyHistory.append([VEnergy + IEnergy, VEnergy, IEnergy])
print VEnergy + IEnergy, '(Total) = ', VEnergy, '(Vector)+', IEnergy, '(Image)'
# plot some stuff
if cf.io.plotEvery > 0 and (((it + 1) % cf.io.plotEvery) == 0
or it == cf.optim.Niter - 1):
GeoRegPlots(subid, cf, p, t, Imsmts, cpinds, cpstates, msmtinds,
gradAtMsmts, EnergyHistory)
# end if
if cf.optim.method == 'FIXEDGD':
# automatic stepsize selection in the first three steps
if it == 1:
# TODO: BEWARE There are hardcoded numbers here for 2D and 3D
#first find max absolute value across voxels in gradient
temp = ca.Field3D(grad_m.grid(), ca.MEM_HOST)
ca.Copy(temp, grad_m)
temp_x, temp_y, temp_z = temp.asnp()
temp1 = np.square(temp_x.flatten()) + np.square(
temp_y.flatten()) + np.square(temp_z.flatten())
medianval = np.median(temp1[temp1 > 0.0000000001])
del temp, temp1, temp_x, temp_y, temp_z
#2D images for 2000 iters
#p.stepSize = float(0.000000002*medianval)
#3D images for 2000 iters
p.stepSize = float(0.000002 * medianval)
print 'rank:', Compute.GetMPIInfo(
)['rank'], ', localRank:', Compute.GetMPIInfo(
)['local_rank'], 'subid: ', subid, ' Selecting initial step size in the beginning to be ', str(
p.stepSize)
if it > 3:
totalEnergyDiff = EnergyHistory[-1][0] - EnergyHistory[-2][0]
if totalEnergyDiff > 0.0:
if cf.optim.maxPert is not None:
print 'rank:', Compute.GetMPIInfo(
)['rank'], ', localRank:', Compute.GetMPIInfo(
)['local_rank'], 'subid: ', subid, ' Reducing stepsize for gradient descent by ', str(
cf.optim.maxPert *
100), '%. The new step size is ', str(
p.stepSize * (1 - cf.optim.maxPert))
p.stepSize = p.stepSize * (1 - cf.optim.maxPert)
# take gradient descent step
ca.Add_MulC_I(p.m0, grad_m, -p.stepSize)
else:
raise Exception("Unknown optimization scheme: " + cf.optim.optMethod)
# end if
# now divide to get new base image
ca.Div(p.I0, sumSplatI, sumJac)
return (EnergyHistory)
