def produceAlignedImagesStep(self, volumeIsCTFCorrected, fn, images):
from numpy import array, dot
fnOut = 'classes_aligned@' + fn
MDin = emlib.MetaData(images)
MDout = emlib.MetaData()
n = 1
hasCTF = MDin.containsLabel(emlib.MDL_CTF_MODEL)
for i in MDin:
fnImg = MDin.getValue(emlib.MDL_IMAGE,i)
fnImgRef = MDin.getValue(emlib.MDL_IMAGE_REF,i)
maxCC = MDin.getValue(emlib.MDL_MAXCC,i)
rot = MDin.getValue(emlib.MDL_ANGLE_ROT,i)
tilt = MDin.getValue(emlib.MDL_ANGLE_TILT,i)
psi =-1.*MDin.getValue(emlib.MDL_ANGLE_PSI,i)
flip = MDin.getValue(emlib.MDL_FLIP,i)
if flip:
psi = -psi
eulerMatrix = emlib.Euler_angles2matrix(0., 0., psi)
x = MDin.getValue(emlib.MDL_SHIFT_X,i)
y = MDin.getValue(emlib.MDL_SHIFT_Y,i)
shift = array([x, y, 0])
shiftOut = dot(eulerMatrix, shift)
[x,y,z]= shiftOut
if flip:
x = -x
id = MDout.addObject()
MDout.setValue(emlib.MDL_IMAGE, fnImg, id)
MDout.setValue(emlib.MDL_IMAGE_REF, fnImgRef, id)
MDout.setValue(emlib.MDL_IMAGE1, "%05d@%s"%(n, self._getExtraPath("diff.stk")), id)
if hasCTF:
fnCTF = MDin.getValue(emlib.MDL_CTF_MODEL,i)
MDout.setValue(emlib.MDL_CTF_MODEL,fnCTF,id)
MDout.setValue(emlib.MDL_MAXCC, maxCC, id)
MDout.setValue(emlib.MDL_ANGLE_ROT, rot, id)
MDout.setValue(emlib.MDL_ANGLE_TILT, tilt, id)
MDout.setValue(emlib.MDL_ANGLE_PSI, psi, id)
MDout.setValue(emlib.MDL_SHIFT_X, x,id)
MDout.setValue(emlib.MDL_SHIFT_Y, y,id)
MDout.setValue(emlib.MDL_FLIP,flip,id)
MDout.setValue(emlib.MDL_ENABLED,1,id)
n+=1
MDout.write(fnOut,emlib.MD_APPEND)
# Actually create the differences
img = emlib.Image()
imgRef = emlib.Image()
if hasCTF and volumeIsCTFCorrected:
Ts = MDin.getValue(emlib.MDL_SAMPLINGRATE, MDin.firstObject())
for i in MDout:
img.readApplyGeo(MDout,i)
imgRef.read(MDout.getValue(emlib.MDL_IMAGE_REF,i))
if hasCTF and volumeIsCTFCorrected:
fnCTF = MDout.getValue(emlib.MDL_CTF_MODEL,i)
emlib.applyCTF(imgRef, fnCTF, Ts)
img.convert2DataType(emlib.DT_DOUBLE)
imgDiff = img-imgRef
imgDiff.write(MDout.getValue(emlib.MDL_IMAGE1,i))
def convertInputStep(self, volName):
# Read volume and convert to DOUBLE
self.vol = emlib.Image(volName)
self.vol.convert2DataType(emlib.DT_DOUBLE)
# Mask volume if needed
if self.refMask.get() is not None:
maskName = getImageLocation(self.refMask.get())
self.mask = emlib.Image(maskName)
self.mask.convert2DataType(emlib.DT_DOUBLE)
self.vol.inplaceMultiply(self.mask)
padding = 2
maxFreq = 0.5
splineDegree = 3
###
self.fourierProjectVol = emlib.FourierProjector(
self.vol, padding, maxFreq, splineDegree)
###
partSet = self.inputParticles.get()
nPart = len(partSet)
numberOfTasks = min(nPart, max(self.numberOfThreads.get() - 1, 1))
taskSize = int(nPart / numberOfTasks)
md = emlib.MetaData()
# Convert angles and shifts from volume system of coordinates to
# projection system of coordinates
mdCount = 0
for index, part in enumerate(partSet):
objId = md.addObject()
imgRow = emlib.metadata.Row()
particleToRow(part, imgRow)
shifts, angles = geometryFromMatrix(
part.getTransform().getMatrix(), True)
imgRow.setValue(emlib.MDL_SHIFT_X, -shifts[0])
imgRow.setValue(emlib.MDL_SHIFT_Y, -shifts[1])
imgRow.setValue(emlib.MDL_SHIFT_Z, 0.)
imgRow.setValue(emlib.MDL_ANGLE_ROT, angles[0])
imgRow.setValue(emlib.MDL_ANGLE_TILT, angles[1])
imgRow.setValue(emlib.MDL_ANGLE_PSI, angles[2])
imgRow.writeToMd(md, objId)
# Write a new metadata every taskSize number of elements
# except in the last chunk where we want to add also the
# remainder and the condition is the last element
if ((index % taskSize == taskSize - 1
and mdCount < numberOfTasks - 1) or index == nPart - 1):
md.write(self._getInputParticlesSubsetFn(mdCount))
md.clear()
mdCount += 1
x, y, _ = partSet.getDim()
emlib.createEmptyFile(self._getProjGalleryFn(), x, y, 1, nPart)
def _processMovie(self, movie):
movieId = movie.getObjId()
if not self.doGainProcess(movieId):
return
inputGain = self.getInputGain()
if self.estimateGain.get() and not movieId in self.estimatedIds:
self.estimatedIds.append(movieId)
self.estimateGainFun(movie)
if self.estimateResidualGain.get(
) and not movieId in self.estimatedResIds:
print('\nEstimating residual gain')
self.estimatedResIds.append(movieId)
self.estimateGainFun(movie, residual=True)
# If the gain hasn't been oriented or normalized, we still need orientedGain
if not os.path.exists(self.getOrientedGainPath()):
# No previous gain: orientedGain is the estimated
if not inputGain is None:
G = emlib.Image()
G.read(inputGain)
G.write(self.getOrientedGainPath())
fnSummary = self._getPath("summary.txt")
fnMonitorSummary = self._getPath("summaryForMonitor.txt")
if not os.path.exists(fnSummary):
fhSummary = open(fnSummary, "w")
fnMonitorSummary = open(fnMonitorSummary, "w")
else:
fhSummary = open(fnSummary, "a")
fnMonitorSummary = open(fnMonitorSummary, "a")
resid_gain = self.getResidualGainPath(movieId)
if os.path.exists(resid_gain):
G = emlib.Image()
G.read(resid_gain)
mean, dev, min, max = G.computeStats()
Gnp = G.getData()
p = np.percentile(Gnp, [2.5, 25, 50, 75, 97.5])
fhSummary.write(
"movie_%06d_residual: mean=%f std=%f [min=%f,max=%f]\n" %
(movieId, mean, dev, min, max))
fhSummary.write(
" 2.5%%=%f 25%%=%f 50%%=%f 75%%=%f 97.5%%=%f\n" %
(p[0], p[1], p[2], p[3], p[4]))
fhSummary.close()
fnMonitorSummary.write("movie_%06d_residual: %f %f %f %f\n" %
(movieId, dev, p[0], p[4], max))
fnMonitorSummary.close()
def applyTransform(imag_array, M, shape):
''' Apply a transformation(M) to a np array(imag) and return it in a given shape
'''
imag = emlib.Image()
imag.setData(imag_array)
imag = imag.applyWarpAffine(list(M.flatten()), shape, True)
return imag.getData()
def normalize(self, what):
# Get overall minimum and maximum
V = emlib.Image()
minAll = 1e38
maxAll = -1e38
for prot in self.inputRuns:
protId = prot.get().getObjId()
protDir = prot.get()._getPath('')
fnVol = os.path.join(protDir, "extra", "result_%s.mrc" % what)
V.read(fnVol)
_, _, minVal, maxVal = V.computeStats()
minAll = min(minAll, minVal)
maxAll = max(maxAll, maxVal)
# Write the Chimera file
for prot in self.inputRuns:
protId = prot.get().getObjId()
protDir = prot.get()._getPath('')
fnRoot = os.path.relpath(os.path.join(protDir, "extra", "result"),
self._getPath(''))
scriptFile = self._getPath('%d_result_%s_chimera.cxc' %
(protId, what))
fhCmd = open(scriptFile, 'w')
fhCmd.write("open %s\n" % (fnRoot + "_final.mrc"))
fhCmd.write("open %s\n" % (fnRoot + "_%s.mrc" % what))
fhCmd.write("vol #1 hide\n")
fhCmd.write(
"scolor #0 volume #1 cmap rainbow cmapRange %f,%f reverseColors True\n"
% (minAll, maxAll))
fhCmd.close()
def runReconstructionStep(self, iterN, refN, program, method, args, suffix,
**kwargs):
#if input metadata is empty create a Blanck image
reconsXmd = 'reconstructionXmd' + suffix
reconsVol = 'reconstructedFileNamesIters' + suffix
mdFn = self._getFileName(reconsXmd, iter=iterN, ref=refN)
volFn = self._getFileName(reconsVol, iter=iterN, ref=refN)
maskFn = self._getFileName('maskedFileNamesIters', iter=iterN, ref=refN)
if method == "art": #or method == 'fourier':
mpi = 1
threads = 1
else:
mpi = self.numberOfMpi.get()
if self.useGpu.get() and self.numberOfMpi.get() > 1:
mpi = len((self.gpuList.get()).split(',')) + 1
threads = self.numberOfThreads.get()
if self.useGpu.get():
args += ' --thr %d' % threads
if isMdEmpty(mdFn):
img = emlib.Image()
img.read(maskFn)
#(x,y,z,n) = img.getDimensions()
self._log.warning(
"Metadata '%s' is empty. \n Creating a random volume file '%s'" %
(mdFn, volFn))
#createEmptyFile(ReconstructedVolume,x,y,z,n)
img.initRandom()
img.write(volFn)
else:
if method == 'fourier':
if self._fourierMaxFrequencyOfInterest[iterN] == -1:
fourierMaxFrequencyOfInterest = self._getFourierMaxFrequencyOfInterest(
iterN - 1, refN)
fourierMaxFrequencyOfInterest = self.resolSam / fourierMaxFrequencyOfInterest + self._constantToAddToMaxReconstructionFrequency[
iterN]
if fourierMaxFrequencyOfInterest > 0.5:
fourierMaxFrequencyOfInterest = 0.5
elif fourierMaxFrequencyOfInterest < 0.:
fourierMaxFrequencyOfInterest = 0.001
else:
fourierMaxFrequencyOfInterest = self._fourierMaxFrequencyOfInterest[
iterN]
args += ' --max_resolution %s' % fourierMaxFrequencyOfInterest
if mpi > 1:
args += ' --mpi_job_size %s' % self.mpiJobSize.get()
self._log.info(
'*********************************************************************'
)
self._log.info('* Reconstruct volume using %s' % method)
self.runJob(program,
args,
numberOfMpi=mpi,
numberOfThreads=threads,
**kwargs)
def getImageFromPath(imagePath):
""" Read an image using Xmipp, convert to PIL
and then return as expected by Tk.
"""
img = emlib.Image(imagePath)
imgPIL = getPILImage(img)
from PIL import ImageTk
imgTk = ImageTk.PhotoImage(imgPIL)
return imgTk
def createVol(self, volume):
vol = emlib.Image()
vol.setDataType(emlib.DT_FLOAT)
vol.resize(projSize, projSize, projSize)
#vol.initRandom(0., .5, emlib.XMIPP_RND_UNIFORM)
vol.initConstant(0.)
for coor in volume:
vol.setPixel(coor[0], coor[1], coor[2], coor[3],
coor[4]) # coor4 is the pixel value
vol.write(self.volBaseFn)
def normalizeGainStep(self):
gainFn = self.getFinalGainPath()
oriGain = emlib.Image()
oriGain.read(gainFn)
oriArray = oriGain.getData()
# normalize array to mean 1
oriArray = oriArray / np.mean(oriArray)
oriGain.setData(oriArray)
oriGain.write(self.getFinalGainPath())
def createProjection(self, proj, num, baseName):
img = emlib.Image()
img.setDataType(emlib.DT_FLOAT)
img.resize(projSize, projSize)
#img.initRandom(0., 1., emlib.XMIPP_RND_GAUSSIAN)
img.initConstant(0.)
for coor in proj:
value = img.getPixel(coor[0], coor[1], coor[2], coor[3])
img.setPixel(coor[0], coor[1], coor[2], coor[3],
coor[4] + value) # coor4 is the pixel value
img.write("%d@" % num + baseName)
class ImageFileHandler(FileHandler):
_image = emlib.Image()
_index = ''
def _getImageString(self, filename):
if isStandardImage(filename):
return "Image file."
x, y, z, n = emlib.getImageSize(filename)
objType = 'Image'
objFileType = "link" if os.path.islink(filename) else "file"
dimMsg = "*%(objType)s %(objFileType)s*\n dimensions: %(x)d x %(y)d"
expMsg = "Columns x Rows "
if z > 1:
dimMsg += " x %(z)d"
expMsg += " x Slices"
objType = 'Volume'
if n > 1:
dimMsg += " x %(n)d"
expMsg += " x Objects"
objType = 'Stack'
return (dimMsg + "\n" + expMsg) % locals()
def _getImagePreview(self, filename):
# If file size if big
if pwutils.getFileSize(filename) > (pwem.Config.MAX_PREVIEW_FILE_SIZE *
1024 * 1024):
return
dim = 128
if isStandardImage(filename):
self.tkImg = gui.getImage(os.path.abspath(filename),
tkImage=True,
maxheight=dim)
else:
fn = self._index + filename
self.tkImg = getTkImage(self._image, fn, dim)
return self.tkImg
def getFilePreview(self, objFile):
fn = objFile.getPath()
return self._getImagePreview(fn), self._getImageString(fn)
def getFileActions(self, objFile):
from .views import DataView
fn = objFile.getPath()
return [('Open with Xmipp viewer', lambda: DataView(fn).show(),
pwutils.Icon.ACTION_VISUALIZE)]
def createMask(self, _maskName):
vol = emlib.Image()
vol.setDataType(emlib.DT_FLOAT)
vol.resize(projSize, projSize, projSize)
vol.initConstant(0.0) #ROB: not sure this is needed
halfDim = int(projSize / 2)
maskRadius2 = maskRadius * maskRadius
for i in range(-halfDim, halfDim):
for j in range(-halfDim, halfDim):
for k in range(-halfDim, halfDim):
if (i * i + j * j + k * k) < maskRadius2:
vol.setPixel(0, k + halfDim, i + halfDim, j + halfDim,
1.) # coor4 is the pixel value
vol.write(_maskName)
def createRandomMicStep(self, mic):
from pwem import Domain
time.sleep(self.creationInterval.get())
getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin',
doRaise=True).getEnviron
# create image
img = emlib.Image()
img.setDataType(emlib.DT_FLOAT)
img.resize(self.xDim, self.yDim)
img.initRandom(0., 1., emlib.XMIPP_RND_UNIFORM)
baseFn = self._getExtraPath(self._singleImageFn)
img.write(baseFn)
md1 = emlib.MetaData()
md1.setColumnFormat(False)
idctf = md1.addObject()
baseFnCtf = self._getTmpPath("ctf_%d.param" % mic)
baseFnImageCTF = self._getExtraPath("imageCTF_%d.xmp" % mic)
md1.setValue(emlib.MDL_CTF_SAMPLING_RATE, 1., idctf)
md1.setValue(emlib.MDL_CTF_VOLTAGE, 200., idctf)
defocus = 20000 + 10000 * random.random()
udefocus = defocus + 1000 * random.random()
vdefocus = defocus + 1000 * random.random()
if udefocus < vdefocus:
aux = vdefocus
vdefocus = udefocus
udefocus = aux
md1.setValue(emlib.MDL_CTF_DEFOCUSU, udefocus, idctf)
md1.setValue(emlib.MDL_CTF_DEFOCUSV, vdefocus, idctf)
md1.setValue(emlib.MDL_CTF_DEFOCUS_ANGLE, 180.0 * random.random(),
idctf)
md1.setValue(emlib.MDL_CTF_CS, 2., idctf)
md1.setValue(emlib.MDL_CTF_Q0, 0.07, idctf)
md1.setValue(emlib.MDL_CTF_K, 1., idctf)
md1.write(baseFnCtf)
# apply ctf
args = " -i %s" % baseFn
args += " -o %s" % baseFnImageCTF
args += " -f ctf %s" % baseFnCtf
args += " --sampling %f" % self.samplingRate
self.runJob("xmipp_transform_filter", args, env=getEnviron())
self.dictObj[baseFnImageCTF] = True
self._checkProcessedData()
def initVolumeData(self, volfile):
if volfile.endswith('.mrc'):
volfile += ':mrc'
if volfile is None:
raise ValueError(volfile)
if '@' in volfile:
[index, file] = volfile.split('@')
else:
file = volfile
if ':' in file:
file = file[0: file.rfind(':')]
if not os.path.exists(file):
raise Exception("File %s does not exists" % file)
self.voxelSize = self.kwargs.get('voxelSize', 1.0)
self.image = emlib.Image(self.volfile)
self.image.convert2DataType(md.DT_DOUBLE)
self.xdim, self.ydim, self.zdim, self.n = self.image.getDimensions()
self.vol = self.image.getData()
def projectStep(self, start, end, samplingRate, threadNumber):
# Project
md = emlib.MetaData(self._getInputParticlesSubsetFn(threadNumber))
##
projection = emlib.Image()
projection.setDataType(emlib.DT_DOUBLE)
##
for id in md:
rot = md.getValue(emlib.MDL_ANGLE_ROT, id)
tilt = md.getValue(emlib.MDL_ANGLE_TILT, id)
psi = md.getValue(emlib.MDL_ANGLE_PSI, id)
##projection =self.vol.projectVolumeDouble(rot, tilt, psi)
self.fourierProjectVol.projectVolume(projection, rot, tilt, psi)
##
# Apply CTF
if self.projType == self.CORRECT_NONE:
pass
elif self.projType == self.CORRECT_FULL_CTF:
emlib.applyCTF(projection, md, samplingRate, id, False)
elif self.projType == self.CORRECT_PHASE_FLIP:
emlib.applyCTF(projection, md, samplingRate, id, True)
else:
raise Exception("ERROR: Unknown projection mode: %d" %
self.projType)
# Shift image
projection.applyGeo(md, id, True, False) #onlyapplyshist, wrap
ih = ImageHandler()
expProj = ih.read(md.getValue(emlib.MDL_IMAGE, id))
expProj.convert2DataType(emlib.DT_DOUBLE)
# Subtract from experimental and write result
projection.resetOrigin()
if self.normalize:
expProj = expProj.adjustAndSubtract(projection)
else:
expProj.inplaceSubtract(projection)
expProj.write(self._getProjGalleryIndexFn(id + start - 1))
def _processRows(self, label, fnExp, mdIn, mdExp, Nrepeats, fileLabels):
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
maxPsi = 180
maxShift = round(newXdim / 10)
for row in iterRows(mdIn):
fnImg = row.getValue(emlib.MDL_IMAGE)
myRow = row
I = emlib.Image(fnImg)
Xdim, Ydim, _, _ = I.getDimensions()
Xdim2 = Xdim / 2
Ydim2 = Ydim / 2
if Nrepeats == 0:
myRow.addToMd(mdExp)
idx += 1
fileLabels.write(str(label - 1) + '\n')
else:
for i in range(Nrepeats):
psiDeg = np.random.uniform(-maxPsi, maxPsi)
psi = psiDeg * math.pi / 180.0
deltaX = np.random.uniform(-maxShift, maxShift)
deltaY = np.random.uniform(-maxShift, maxShift)
c = math.cos(psi)
s = math.sin(psi)
M = np.float32([[c, s, (1 - c) * Xdim2 - s * Ydim2 + deltaX],
[-s, c, s * Xdim2 + (1 - c) * Ydim2 + deltaY]])
newFn = ('%06d@' % idx) + fnExp[:-3] + 'stk'
self._ih.applyTransform(
fnImg, newFn, M, (Ydim, Xdim), doWrap=True)
myRow.setValue(emlib.MDL_IMAGE, newFn)
myRow.setValue(emlib.MDL_ANGLE_PSI, psiDeg)
myRow.setValue(emlib.MDL_SHIFT_X, deltaX)
myRow.setValue(emlib.MDL_SHIFT_Y, deltaY)
myRow.addToMd(mdExp)
idx += 1
fileLabels.write(str(label - 1) + '\n')
def testCtfConsensus1(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
# create one fake micrographs image
projSize = 32
img = emlib.Image()
img.setDataType(emlib.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 4000 + 10 * i
defocusV = 4000 + i
defocusAngle = i * 10
resolution = 2
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle,
resolution, psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
ctfSet1.write()
micSet.write()
# import micrograph set
args = {
'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
self.launchProtocol(protCTF1)
# launch CTF consensus protocol
protCtfConsensus = self.newProtocol(XmippProtCTFConsensus)
protCtfConsensus.inputCTF.set(protCTF1.outputCTF)
protCtfConsensus.setObjLabel('ctf consensus')
self.launchProtocol(protCtfConsensus)
self.checkOutputSize(protCtfConsensus)
ctf0 = protCtfConsensus.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 4010)
def readImage(fn):
img = emlib.Image()
img.read(fn)
return img
def writeImageFromArray(array, fn):
img = emlib.Image()
img.setData(array)
img.write(fn)
def copy_image(imag):
''' Return a copy of a xmipp_image
'''
new_imag = emlib.Image()
new_imag.setData(imag.getData())
return new_imag
