def createOutputStep(self):
cleanPattern(self._getExtraPath('*.stk'))
cleanPattern(self._getExtraPath('projectionsCudaCorr*'))
inputParticles = self.inputSet.get()
fnOutputParticles = self._getExtraPath('outConesParticles.xmd')
outputSetOfParticles = self._createSetOfParticles()
outputSetOfParticles.copyInfo(inputParticles)
outputSetOfParticles.setAlignmentProj()
Xdim = inputParticles.getXDim()
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
Ts = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
if newXdim != Xdim:
self.scaleFactor = Ts / inputParticles.getSamplingRate()
self.iterMd = md.iterRows(fnOutputParticles, emlib.MDL_ITEM_ID)
self.lastRow = next(self.iterMd)
outputSetOfParticles.copyItems(inputParticles,
updateItemCallback=self._updateItem)
else:
readSetOfParticles(fnOutputParticles, outputSetOfParticles)
self._defineOutputs(outputParticles=outputSetOfParticles)
def predictStep(self, gpuId):
if not exists(self._getConePrediction()):
# if self.useQueueForSteps() or self.useQueue():
# myStr = os.environ["CUDA_VISIBLE_DEVICES"]
# else:
# myStr = self.gpuList.get()
# numGPU = myStr.split(',')
# numGPU = numGPU[0]
# print("Predict", myStr, numGPU)
# sys.stdout.flush()
mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc))
self.numCones = mdNumCones.size()
imgsOutStk = self._getExtraPath('images_out_filtered.stk')
imgsOutXmd = self._getExtraPath('images_out_filtered.xmd')
self.runJob("xmipp_transform_filter", " -i %s -o %s "
"--save_metadata_stack %s "
"--keep_input_columns "
"--fourier low_pass %f " %
(self.imgsFn, imgsOutStk, imgsOutXmd, 0.15), numberOfMpi=self.myMPI.get())
numMax = int(self.numConesSelected)
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
args = "%s %s %d %d %d " % (imgsOutXmd, self._getExtraPath(), newXdim, self.numCones, numMax)
#args += " %(GPU)s"
args += " %s "%(gpuId)
self.runJob("xmipp_cone_deepalign_predict", args, numberOfMpi=1, env=self.getCondaEnv())
def alignmentStep(self):
imgsFn = self._getFileName('imgsFn')
fnVol = self._getFileName('fnVol')
fnOut = self._getFileName('fnOut')
fnVolMask = self._getFileName('fnVolMask')
fnOutDir = self._getFileName('fnOutDir')
Ts = readInfoField(self._getExtraPath(), "sampling",
md.MDL_SAMPLINGRATE)
params = ' -i %s --ref %s -o %s --optimizeDeformation --optimizeDefocus ' \
'--l1 %d --l2 %d --max_shift %f --max_angular_change %f --sampling %f ' \
' --max_resolution %f --odir %s --resume --regularization %f' %\
(imgsFn, fnVol, fnOut, self.l1.get(), self.l2.get(), self.maxShift,
self.maxAngular, Ts, self.maxResolution, fnOutDir, self.regularization.get())
if self.optimizeAlignment.get():
params += ' --optimizeAlignment'
if self.ignoreCTF.get():
params += ' --ignoreCTF'
if self.inputParticles.get().isPhaseFlipped():
params += ' --phaseFlipped'
if self.inputVolumeMask.get():
params += ' --mask %s' % fnVolMask
if self.useGpu.get():
params += ' --device %d' % self.getGpuList()[0]
program = 'xmipp_cuda_angular_sph_alignment'
self.runJob(program, params)
else:
program = 'xmipp_angular_sph_alignment'
self.runJob(program, params, numberOfMpi=self.numberOfMpi.get())
def convertStep(self):
if self.modelPretrain.get() is True:
fnPreProtocol = self.pretrainedModels.get()._getExtraPath()
preXDim = readInfoField(fnPreProtocol, "size", emlib.MDL_XSIZE)
self.inputTrainSet = self.pretrainedModels.get().inputTrainSet
from ..convert import writeSetOfParticles
inputParticles = self.inputSet.get()
writeSetOfParticles(inputParticles, self.imgsFn)
Ts = inputParticles.getSamplingRate()
row = getFirstRow(self.imgsFn)
hasCTF = row.containsLabel(emlib.MDL_CTF_DEFOCUSU) or emlib.containsLabel(
emlib.MDL_CTF_MODEL)
fnCorrected = self.imgsFn
self._correctWiener(hasCTF, self.imgsFn, Ts)
Xdim = inputParticles.getXDim()
newTs = self.targetResolution.get() * 1.0 / 3.0
newTs = max(Ts, newTs)
self.newXdim = int(preXDim) if self.modelPretrain.get() else int(
float(Xdim * Ts / newTs))
self.firstMaxShift = int(round(self.newXdim / 10))
writeInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE, newTs)
writeInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE,
self.newXdim)
self._resize(Xdim, fnCorrected, 'scaled_particles', self.imgsFn)
self._removeCorrectedParticles()
fnVol = self._getTmpPath(self._fnVolumeVol)
self._ih.convert(self.inputVolume.get(), fnVol)
Xdim = self.inputVolume.get().getDim()[0]
if Xdim != self.newXdim:
self.runJob("xmipp_image_resize",
"-i %s --fourier %d" % (fnVol, self.newXdim),
numberOfMpi=self.myMPI.get())
inputTrain = self.inputTrainSet.get()
writeSetOfParticles(inputTrain, self.trainImgsFn)
row = getFirstRow(self.trainImgsFn)
hasCTF = row.containsLabel(emlib.MDL_CTF_DEFOCUSU) or emlib.containsLabel(
emlib.MDL_CTF_MODEL)
fnCorrected = self.trainImgsFn
self._correctWiener(hasCTF, self.trainImgsFn, Ts)
self._resize(Xdim, fnCorrected,
'scaled_train_particles', self.trainImgsFn)
self._removeCorrectedParticles()
def splitVolumeStep(self):
newTs = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE)
fnMask = ""
if self.mask.hasValue():
fnMask = self._getExtraPath("mask.vol")
img = ImageHandler()
img.convert(self.mask.get(), fnMask)
self.runJob('xmipp_image_resize',
"-i %s --dim %d" % (fnMask, newXdim),
numberOfMpi=1)
self.runJob('xmipp_transform_threshold',
"-i %s --select below 0.5 --substitute binarize" %
fnMask,
numberOfMpi=1)
args = "-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f" % \
(self._getDirectionalClassesFn(), self._getExtraPath("split"),
self.Nrec.get(), self.Nsamples.get(),
self.symmetryGroup.get(), self.alpha.get())
if fnMask != "":
args += " --mask binary_file %s" % fnMask
self.runJob("xmipp_classify_first_split", args, numberOfMpi=1)
def _coneStep(self, gpuId, idx, modelFn):
fnLabels = self._getExtraPath('labels.txt')
fileLabels = open(fnLabels, "r")
expSet = self._getProjectionsExp(self.numCones)
if not exists(expSet):
for n in range(1, self.numCones):
if exists(self._getProjectionsExp(self.numCones - n)):
expSet = self._getProjectionsExp(self.numCones - n)
break
newFnLabels = self._getExtraPath('labels%d.txt' % idx)
newFileLabels = open(newFnLabels, "w")
lines = fileLabels.readlines()
for line in lines:
if line == str(idx - 1) + '\n':
newFileLabels.write('1\n')
else:
newFileLabels.write('0\n')
newFileLabels.close()
fileLabels.close()
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
fnLabels = self._getExtraPath('labels%d.txt' % idx)
print("Training region ", idx, " in GPU ", gpuId)
sys.stdout.flush()
try:
args = "%s %s %s %s %d %d %d %d " % (
expSet, fnLabels, self._getExtraPath(),
modelFn+'_aux', self.numEpochs, newXdim, 2, self.batchSize.get())
#args += " %(GPU)s"
args += " %s " % (gpuId)
#args += " %s " %(int(idx % totalGpu))
self.runJob("xmipp_cone_deepalign", args, numberOfMpi=1, env=self.getCondaEnv())
except Exception as e:
raise Exception(
"ERROR: Please, if you are suffering memory problems, "
"check the target resolution to work with lower dimensions.")
moveFile(self._getExtraPath(modelFn + '_aux.h5'), self._getExtraPath(modelFn + '.h5'))
def _processRows(self, label, fnExp, mdIn, mdExp, Nrepeats, fileLabels):
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
maxPsi = 180
maxShift = round(newXdim / 10)
for row in iterRows(mdIn):
fnImg = row.getValue(emlib.MDL_IMAGE)
myRow = row
I = emlib.Image(fnImg)
Xdim, Ydim, _, _ = I.getDimensions()
Xdim2 = Xdim / 2
Ydim2 = Ydim / 2
if Nrepeats == 0:
myRow.addToMd(mdExp)
idx += 1
fileLabels.write(str(label - 1) + '\n')
else:
for i in range(Nrepeats):
psiDeg = np.random.uniform(-maxPsi, maxPsi)
psi = psiDeg * math.pi / 180.0
deltaX = np.random.uniform(-maxShift, maxShift)
deltaY = np.random.uniform(-maxShift, maxShift)
c = math.cos(psi)
s = math.sin(psi)
M = np.float32([[c, s, (1 - c) * Xdim2 - s * Ydim2 + deltaX],
[-s, c, s * Xdim2 + (1 - c) * Ydim2 + deltaY]])
newFn = ('%06d@' % idx) + fnExp[:-3] + 'stk'
self._ih.applyTransform(
fnImg, newFn, M, (Ydim, Xdim), doWrap=True)
myRow.setValue(emlib.MDL_IMAGE, newFn)
myRow.setValue(emlib.MDL_ANGLE_PSI, psiDeg)
myRow.setValue(emlib.MDL_SHIFT_X, deltaX)
myRow.setValue(emlib.MDL_SHIFT_Y, deltaY)
myRow.addToMd(mdExp)
idx += 1
fileLabels.write(str(label - 1) + '\n')
def projectStep(self, numProj, iniRot, endRot, iniTilt, endTilt, fn, idx):
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
fnVol = self._getTmpPath(self._fnVolumeVol)
uniformProjectionsStr = """
# XMIPP_STAR_1 *
data_block1
_dimensions2D '%d %d'
_projRotRange '%d %d %d'
_projRotRandomness random
_projRotNoise '0'
_projTiltRange '%d %d 1'
_projTiltRandomness random
_projTiltNoise '0'
_projPsiRange '0 0 1'
_projPsiRandomness random
_projPsiNoise '0'
_noisePixelLevel '0'
_noiseCoord '0'
""" % (newXdim, newXdim, iniRot, endRot, numProj, iniTilt, endTilt)
fnParams = self._getExtraPath("uniformProjections%d.xmd" % idx)
fh = open(fnParams, "w")
fh.write(uniformProjectionsStr)
fh.close()
fnProjs = self._getExtraPath(fn + "%d.stk" % idx)
self.runJob("xmipp_phantom_project",
"-i %s -o %s --method fourier 1 0.5 "
"--params %s" % (fnVol, fnProjs, fnParams), numberOfMpi=1)
fnProjsXmd=fnProjs[:-3]+'xmd'
self.runJob("xmipp_metadata_utilities",
"-i %s --fill ref lineal 1 1 " % (fnProjsXmd), numberOfMpi=1)
cleanPattern(self._getExtraPath('uniformProjections*'))
def createOutputStep(self):
inputParticles = self.inputParticles.get()
if not self._useSeveralClasses():
newTs = inputParticles.getSamplingRate()
else:
newTs = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
self.mdClasses = emlib.MetaData(self._getDirectionalClassesFn())
self.mdImages = emlib.MetaData(self._getDirectionalImagesFn())
classes2D = self._createSetOfClasses2D(inputParticles)
classes2D.getImages().setSamplingRate(newTs)
self.averageSet = self._createSetOfAverages()
self.averageSet.copyInfo(inputParticles)
self.averageSet.setAlignmentProj()
self.averageSet.setSamplingRate(newTs)
# Let's use a SetMdIterator because it should be less particles
# in the metadata produced than in the input set
iterator = md.SetMdIterator(self.mdImages,
sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self._updateParticle,
skipDisabled=True)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
if exists(fnHomogeneous):
homogeneousSet = self._createSetOfParticles()
homogeneousSet.copyInfo(inputParticles)
homogeneousSet.setSamplingRate(newTs)
homogeneousSet.setAlignmentProj()
self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID)
self.lastRow = next(self.iterMd)
homogeneousSet.copyItems(
inputParticles, updateItemCallback=self._updateHomogeneousItem)
self._defineOutputs(outputHomogeneous=homogeneousSet)
self._defineSourceRelation(self.inputParticles, homogeneousSet)
classes2D.classifyItems(updateItemCallback=iterator.updateItem,
updateClassCallback=self._updateClass)
self._defineOutputs(outputClasses=classes2D)
self._defineOutputs(outputAverages=self.averageSet)
self._defineSourceRelation(self.inputParticles, classes2D)
self._defineSourceRelation(self.inputParticles, self.averageSet)
if self.splitVolume and self.directionalClasses.get() > 1:
volumesSet = self._createSetOfVolumes()
volumesSet.setSamplingRate(newTs)
for i in range(2):
fnVol = self._getExtraPath("split_v%d.vol" % (i + 1))
fnMrc = self._getExtraPath("split_v%d.mrc" % (i + 1))
self.runJob("xmipp_image_convert",
"-i %s -o %s -t vol" % (fnVol, fnMrc),
numberOfMpi=1)
self.runJob("xmipp_image_header",
"-i %s --sampling_rate %f" % (fnMrc, newTs),
numberOfMpi=1)
cleanPath(fnVol)
vol = Volume()
vol.setLocation(1, fnMrc)
volumesSet.append(vol)
self._defineOutputs(outputVolumes=volumesSet)
self._defineSourceRelation(inputParticles, volumesSet)
def refineAnglesStep(self):
fnTmpDir = self._getTmpPath()
fnDirectional = self._getDirectionalClassesFn()
inputParticles = self.inputParticles.get()
newTs = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE)
# Generate projections
fnGallery = join(fnTmpDir, "gallery.stk")
fnGalleryMd = join(fnTmpDir, "gallery.doc")
fnVol = self._getInputVolFn()
args = "-i %s -o %s --sampling_rate %f --sym %s" % \
(fnVol, fnGallery, 5.0, self.symmetryGroup)
args += " --compute_neighbors --angular_distance -1 --experimental_images %s" % fnDirectional
self.runJob("xmipp_angular_project_library",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
# Global angular assignment
maxShift = 0.15 * newXdim
fnAngles = join(fnTmpDir, "angles_iter001_00.xmd")
if not self.useGpu.get():
args = '-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0' % \
(fnDirectional, fnGalleryMd, maxShift, fnTmpDir)
self.runJob('xmipp_reconstruct_significant',
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
else:
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
args = '-i %s -r %s -o %s --dev %s ' % (fnDirectional, fnGalleryMd,
fnAngles, GpuListCuda)
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --operate drop_column ref" % fnAngles,
numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set join %s ref2" % (fnAngles, fnDirectional),
numberOfMpi=1)
# Local angular assignment
fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
args = "-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 " % \
(fnAngles, fnAnglesLocalStk, newTs, newXdim / 2, 2, fnVol,
self.targetResolution)
args += " --optimizeShift --max_shift %f" % maxShift
args += " --optimizeAngles --max_angular_change %f" % self.angularDistance
self.runJob("xmipp_angular_continuous_assign2",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
moveFile(self._getPath("directional_local_classes.xmd"),
self._getDirectionalClassesFn())
cleanPattern(self._getExtraPath("direction_*"))
def createOutputStep(self):
inputParticles = self.inputParticles.get()
# if not self._useSeveralClasses():
# newTs = inputParticles.getSamplingRate()
# else:
# newTs = readInfoField(self._getExtraPath(), "sampling",
# xmipp.MDL_SAMPLINGRATE)
self.mdClasses = emlib.MetaData(self._getDirectionalClassesFn())
self.mdImages = emlib.MetaData(self._getDirectionalImagesFn())
origTs = inputParticles.getSamplingRate()
lastTs = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
if origTs != lastTs:
newXdim = inputParticles.getXDim()
self.runJob(
"xmipp_image_resize",
"-i %s -o %s --save_metadata_stack %s --fourier %d" %
(self._getDirectionalClassesFn(),
self._getPath("aux_directional_local_classes.stk"),
self._getPath("aux_directional_classes.xmd"), newXdim),
numberOfMpi=1)
from shutil import copy
copy(self._getPath("aux_directional_local_classes.stk"),
self._getPath("directional_local_classes.stk"))
copy(self._getPath("aux_directional_classes.xmd"),
self._getPath("directional_classes.xmd"))
cleanPattern(self._getPath("aux_directional*"))
classes2D = self._createSetOfClasses2D(inputParticles)
#classes2D.getImages().setSamplingRate(newTs)
classes2D.getImages().setSamplingRate(origTs)
self.averageSet = self._createSetOfAverages()
self.averageSet.copyInfo(inputParticles)
self.averageSet.setAlignmentProj()
#self.averageSet.setSamplingRate(newTs)
self.averageSet.setSamplingRate(origTs)
# Let's use a SetMdIterator because it should be less particles
# in the metadata produced than in the input set
iterator = md.SetMdIterator(self.mdImages,
sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self._updateParticle,
skipDisabled=True)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
if exists(fnHomogeneous):
if origTs != lastTs:
newXdim = inputParticles.getXDim()
self.runJob("xmipp_image_resize",
"-i %s --dim %d" % (fnHomogeneous, newXdim),
numberOfMpi=1)
homogeneousSet = self._createSetOfParticles()
homogeneousSet.copyInfo(inputParticles)
#homogeneousSet.getImages().setSamplingRate(newTs)
homogeneousSet.getImages().setSamplingRate(origTs)
homogeneousSet.setAlignmentProj()
self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID)
self.lastRow = next(self.iterMd)
homogeneousSet.copyItems(
inputParticles, updateItemCallback=self._updateHomogeneousItem)
self._defineOutputs(outputHomogeneous=homogeneousSet)
self._defineSourceRelation(self.inputParticles, homogeneousSet)
#AJ testing
#AJ por que desaparece una clase que tiene imagenes asignadas
listRefId = []
for row in md.iterRows(self.mdClasses, emlib.MDL_REF2):
refId = row.getValue(emlib.MDL_REF2, row.getObjId())
if len(listRefId) > 0 and refId != listRefId[-1] + 1:
whereEnd = listRefId[-1] + 1
for i in range(refId - whereEnd):
rowNew = row
rowNew.setValue(emlib.MDL_REF2, listRefId[-1] + i + 1)
rowNew.setValue(emlib.MDL_IMAGE, 'None')
rowNew.setValue(emlib.MDL_IMAGE1, 'None')
rowNew.addToMd(self.mdClasses)
listRefId.append(listRefId[-1] + i + 1)
listRefId.append(refId)
else:
listRefId.append(refId)
self.mdClasses.write(self._getDirectionalClassesFn())
self.mdClasses = emlib.MetaData(self._getDirectionalClassesFn())
#END AJ
classes2D.classifyItems(updateItemCallback=iterator.updateItem,
updateClassCallback=self._updateClass)
self._defineOutputs(outputClasses=classes2D)
self._defineOutputs(outputAverages=self.averageSet)
self._defineSourceRelation(self.inputParticles, classes2D)
self._defineSourceRelation(self.inputParticles, self.averageSet)
if self.splitVolume and self.directionalClasses.get() > 1:
volumesSet = self._createSetOfVolumes()
#volumesSet.setSamplingRate(newTs)
volumesSet.setSamplingRate(origTs)
for i in range(2):
vol = Volume()
fnVol = self._getExtraPath("split%d.vol" % (i + 1))
if origTs != lastTs:
newXdim = inputParticles.getXDim()
self.runJob("xmipp_image_resize",
"-i %s --dim %d" % (fnVol, newXdim),
numberOfMpi=1)
fnMrc = self._getExtraPath("split%d.mrc" % (i + 1))
self.runJob("xmipp_image_convert",
"-i %s -o %s -t vol" % (fnVol, fnMrc),
numberOfMpi=1)
self.runJob("xmipp_image_header",
"-i %s --sampling_rate %f" % (fnMrc, origTs),
numberOfMpi=1)
cleanPath(fnVol)
vol.setLocation(1, fnMrc)
volumesSet.append(vol)
self._defineOutputs(outputVolumes=volumesSet)
self._defineSourceRelation(inputParticles, volumesSet)