def createOutputStep(self):
cleanPattern(self._getExtraPath('*.stk'))
cleanPattern(self._getExtraPath('projectionsCudaCorr*'))
inputParticles = self.inputSet.get()
fnOutputParticles = self._getExtraPath('outConesParticles.xmd')
outputSetOfParticles = self._createSetOfParticles()
outputSetOfParticles.copyInfo(inputParticles)
outputSetOfParticles.setAlignmentProj()
Xdim = inputParticles.getXDim()
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
Ts = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
if newXdim != Xdim:
self.scaleFactor = Ts / inputParticles.getSamplingRate()
self.iterMd = md.iterRows(fnOutputParticles, emlib.MDL_ITEM_ID)
self.lastRow = next(self.iterMd)
outputSetOfParticles.copyItems(inputParticles,
updateItemCallback=self._updateItem)
else:
readSetOfParticles(fnOutputParticles, outputSetOfParticles)
self._defineOutputs(outputParticles=outputSetOfParticles)
def _mergeAllParFiles(self, iterN, numberOfBlocks):
""" This method merge all parameters files that has been
created in a refineStep. """
self._enterDir(self._getExtraPath())
outFn = self._getFileName('iter_par', iter=iterN)
if numberOfBlocks != 1:
f1 = open(outFn, 'w+')
f1.write(
"C PSI THETA PHI SHX SHY MAG "
"FILM DF1 DF2 ANGAST PSHIFT OCC LogP"
" SIGMA SCORE CHANGE\n")
for block in range(1, numberOfBlocks + 1):
parFn = self._getFileName('iter_par_block',
iter=iterN,
block=block)
if not os.path.exists(parFn):
raise FileNotFoundError("Error: file %s does not exist" %
parFn)
f2 = open(parFn)
for l in f2:
if not l.startswith('C'):
f1.write(l)
f2.close()
cleanPattern(parFn)
f1.close()
else:
parFn = self._getFileName('iter_par_block', iter=iterN, block=1)
moveFile(parFn, outFn)
self._leaveDir()
def refineDefocus(self):
"""compute local defocus using Xmipp (xmipp_angular_continuous_assign2) and add to metadata columns related to defocus"""
fnVol = self._getExtraPath("volume.vol")
fnIn = self._getExtraPath('input_imgs.xmd')
fnOut = self._getExtraPath('output_imgs.xmd')
anglesOutFn = self._getExtraPath("anglesCont.stk")
Ts = self.inputSet.get().getSamplingRate()
args="-i %s -o %s --ref %s --optimizeDefocus --max_defocus_change %d --sampling %f "\
"--optimizeGray --max_gray_scale %f --max_gray_shift %f"%\
(fnIn,anglesOutFn,fnVol,self.maxDefocusChange.get(),Ts,self.maxGrayScaleChange.get(),
self.maxGrayShiftChange.get())
if self.inputSet.get().isPhaseFlipped():
args += " --phaseFlipped"
self.runJob("xmipp_angular_continuous_assign2", args)
fnCont = self._getExtraPath('anglesCont.xmd')
self.runJob(
"xmipp_metadata_utilities",
'-i %s --operate keep_column "itemId ctfDefocusU ctfDefocusV ctfDefocusChange ctfDefocusAngle"'
% fnCont,
numberOfMpi=1)
self.runJob(
"xmipp_metadata_utilities",
'-i %s -o %s --operate drop_column "ctfDefocusU ctfDefocusV ctfDefocusChange ctfDefocusAngle"'
% (fnIn, fnOut),
numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set join %s itemId itemId" % (fnOut, fnCont),
numberOfMpi=1)
cleanPattern(self._getExtraPath("anglesCont.*"))
def reconstructVolumeStep(self, args):
""" Run the EMAN program to reconstruct a volume. """
cleanPattern(self._getFileName("volume"))
if self.useE2make3d:
program = Plugin.getProgram('e2make3d.py')
else:
program = Plugin.getProgram('e2make3dpar.py')
self.runJob(program, args, cwd=self._getExtraPath(), numberOfThreads=1)
def refineStep(self, args):
""" Run the EMAN program to refine a volume. """
if not self.doContinue:
cleanPattern(self._getExtraPath('refine_01'))
program = eman2.Plugin.getProgram('e2refine_easy.py')
# mpi and threads are handled by EMAN itself
self.runJob(program, args, cwd=self._getExtraPath(),
numberOfMpi=1, numberOfThreads=1)
def createInitialModelStep(self, args):
""" Run the EMAN program to create the initial model. """
cleanPattern(self._getExtraPath('initial_models'))
if self._isHighSym():
program = eman2.getEmanProgram('e2initialmodel_hisym.py')
else:
program = eman2.getEmanProgram('e2initialmodel.py')
self.runJob(program, args, cwd=self._getExtraPath())
def createInitialModelStep(self, args):
""" Run the EMAN program to create the initial model. """
cleanPattern(self._getExtraPath('initmodel_??'))
program = Plugin.getProgram('e2initialmodel_sgd.py')
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
def dockStep(self, fnGridDir, fnSmall):
fnReceptor = os.path.join(fnGridDir, "atomStruct.pdbqt")
fnBase = os.path.splitext(os.path.split(fnSmall)[1])[0]
fnSmallDir = self._getExtraPath(fnBase)
makePath(fnSmallDir)
fnDPF = os.path.join(fnSmallDir, fnBase + ".dpf")
args = " -l %s -r %s -o %s" % (fnSmall, fnReceptor, fnDPF)
args += " -p ga_pop_size=%d" % self.gaPop.get()
args += " -p ga_num_evals=%d" % self.gaNumEvals.get()
args += " -p ga_num_generations=%d" % self.gaNumGens.get()
args += " -p ga_elitism=%d" % self.gaElitism.get()
args += " -p ga_mutation_rate=%f" % self.gaMutationRate.get()
args += " -p ga_crossover_rate=%f" % self.gaCrossOverRate.get()
args += " -p ga_window_size=%d" % self.gaWindowSize.get()
args += " -p sw_max_its=%d" % self.swMaxIts.get()
args += " -p sw_max_succ=%d" % self.swMaxSucc.get()
args += " -p sw_max_fail=%d" % self.swMaxFail.get()
args += " -p sw_rho=%f" % self.swRho.get()
args += " -p sw_lb_rho=%d" % self.swLbRho.get()
args += " -p ls_search_freq=%f" % self.lsFreq.get()
args += " -p ga_run=%d" % self.gaRun.get()
args += " -p rmstol=%f" % self.rmsTol.get()
self.runJob(
bioinformatics_plugin.getMGLPath('bin/pythonsh'),
bioinformatics_plugin.getADTPath('Utilities24/prepare_dpf42.py') +
args)
fnSmallLocal = os.path.split(fnSmall)[1]
createLink(fnSmall, os.path.join(fnSmallDir, fnSmallLocal))
createLink(fnReceptor, os.path.join(fnSmallDir, "atomStruct.pdbqt"))
args = " -r atomStruct.pdbqt -l %s -o library.gpf" % fnSmallLocal
self.runJob(
bioinformatics_plugin.getMGLPath('bin/pythonsh'),
bioinformatics_plugin.getADTPath('Utilities24/prepare_gpf4.py') +
args,
cwd=fnSmallDir)
args = "-p library.gpf -l library.glg"
self.runJob(bioinformatics_plugin.getAutodockPath("autogrid4"),
args,
cwd=fnSmallDir)
args = "-p %s.dpf -l %s.dlg" % (fnBase, fnBase)
self.runJob(bioinformatics_plugin.getAutodockPath("autodock4"),
args,
cwd=fnSmallDir)
# Clean a bit
cleanPattern(os.path.join(fnSmallDir, "atomStruct.*.map"))
def createInitialModelStep(self, args):
""" Run the EMAN program to create the initial model. """
cleanPattern(self._getExtraPath('initial_models'))
if self._isHighSym():
program = Plugin.getProgram('e2initialmodel_hisym.py')
else:
program = Plugin.getProgram('e2initialmodel.py')
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
def convertOutput(self):
program = emantomo.Plugin.getProgram('e2proc3d.py')
for hdfFile in glob.glob(self._getExtraPath('*.hdf')):
args = ' --unstacking'
args += ' %s' % abspath(hdfFile)
args += ' %s' % abspath(
self._getExtraPath(pwutils.replaceBaseExt(hdfFile, 'mrc')))
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
cleanPattern(hdfFile)
def convertToPseudoAtomsStep(self, inputFn, fnMask, sampling, prefix=''):
pseudoatoms = 'pseudoatoms%s'%prefix
outputFn = self._getPath(pseudoatoms)
sigma = sampling * self.pseudoAtomRadius.get()
targetErr = self.pseudoAtomTarget.get()
nthreads = self.numberOfThreads.get()
params = "-i %(inputFn)s -o %(outputFn)s --sigma %(sigma)f --thr %(nthreads)d "
params += "--targetError %(targetErr)f --sampling_rate %(sampling)f -v 2 --intensityColumn Bfactor"
if fnMask:
params += " --mask binary_file %(fnMask)s"
print params%locals()
self.runJob("xmipp_volume_to_pseudoatoms", params % locals())
for suffix in ["_approximation.vol", "_distance.hist"]:
moveFile(self._getPath(pseudoatoms+suffix), self._getExtraPath(pseudoatoms+suffix))
cleanPattern(self._getPath(pseudoatoms+'_*'))
def gatherResultsStep(self, debugging):
self._writeFreqsMetaData("fraction*_freq.txt",
self._defineResultsTxt())
if self.doNoise.get():
self._writeFreqsMetaData("Nfraction*_freq.txt",
self._defineResultsNoiseTxt())
if not debugging:
#cleanPattern(self._getExtraPath("fraction*_freq.txt")) AJ volver
cleanPattern(self._getExtraPath('newImages.stk'))
cleanPattern(self._getExtraPath('newImages.xmd'))
if self.doNoise.get():
cleanPattern(self._getExtraPath("Nfraction*_freq.txt"))
cleanPattern(self._getExtraPath('Ref_Projections*'))
cleanPattern(self._getExtraPath('newVolume.vol'))
def convertToPseudoAtomsStep(self, inputFn, fnMask, sampling, prefix=''):
pseudoatoms = 'pseudoatoms%s' % prefix
outputFn = self._getPath(pseudoatoms)
sigma = sampling * self.pseudoAtomRadius.get()
targetErr = self.pseudoAtomTarget.get()
nthreads = self.numberOfThreads.get()
params = "-i %(inputFn)s -o %(outputFn)s --sigma %(sigma)f --thr %(nthreads)d "
params += "--targetError %(targetErr)f --sampling_rate %(sampling)f -v 2 --intensityColumn Bfactor"
if fnMask:
params += " --mask binary_file %(fnMask)s"
print params % locals()
self.runJob("xmipp_volume_to_pseudoatoms", params % locals())
for suffix in ["_approximation.vol", "_distance.hist"]:
moveFile(self._getPath(pseudoatoms + suffix),
self._getExtraPath(pseudoatoms + suffix))
cleanPattern(self._getPath(pseudoatoms + '_*'))
def refineAnglesStep(self):
fnTmpDir = self._getTmpPath()
fnDirectional = self._getDirectionalClassesFn()
inputParticles = self.inputParticles.get()
newTs = self.readInfoField(self._getExtraPath(), "sampling",
xmippLib.MDL_SAMPLINGRATE)
newXdim = self.readInfoField(self._getExtraPath(), "size",
xmippLib.MDL_XSIZE)
# Generate projections
fnGallery = join(fnTmpDir, "gallery.stk")
fnGalleryMd = join(fnTmpDir, "gallery.doc")
fnVol = self._getInputVolFn()
args = "-i %s -o %s --sampling_rate %f --sym %s" % \
(fnVol, fnGallery, 5.0, self.symmetryGroup)
args += " --compute_neighbors --angular_distance -1 --experimental_images %s" % fnDirectional
self.runJob("xmipp_angular_project_library", args,
numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get())
# Global angular assignment
maxShift = 0.15 * newXdim
args = '-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0' % \
(fnDirectional, fnGalleryMd, maxShift, fnTmpDir)
self.runJob('xmipp_reconstruct_significant', args,
numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get())
fnAngles = join(fnTmpDir, "angles_iter001_00.xmd")
self.runJob("xmipp_metadata_utilities",
"-i %s --operate drop_column ref" % fnAngles, numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set join %s ref2" % (fnAngles, fnDirectional),
numberOfMpi=1)
# Local angular assignment
fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
args = "-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 --Nsimultaneous %d" % \
(fnAngles, fnAnglesLocalStk, newTs, newXdim / 2, 2, fnVol,
self.targetResolution, 8)
args += " --optimizeShift --max_shift %f" % maxShift
args += " --optimizeAngles --max_angular_change %f" % self.angularDistance
self.runJob("xmipp_angular_continuous_assign2", args,
numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get())
moveFile(self._getPath("directional_local_classes.xmd"),
self._getDirectionalClassesFn())
cleanPattern(self._getExtraPath("direction_*"))
def ransacIterationStep(self, n):
fnOutputReducedClass = self._getExtraPath("reducedClasses.xmd")
fnBase = "ransac%05d"%n
fnRoot = self._getTmpPath(fnBase)
if self.dimRed:
# Get a random sample of images
self.runJob("xmipp_transform_dimred","-i %s --randomSample %s.xmd %d -m LTSA "%(fnOutputReducedClass,fnRoot,self.numGrids.get()))
else:
self.runJob("xmipp_metadata_utilities","-i %s -o %s.xmd --operate random_subset %d --mode overwrite "%(fnOutputReducedClass,fnRoot,self.numSamples.get()))
self.runJob("xmipp_metadata_utilities","-i %s.xmd --fill angleRot rand_uniform -180 180 "%(fnRoot))
self.runJob("xmipp_metadata_utilities","-i %s.xmd --fill angleTilt rand_uniform 0 180 "%(fnRoot))
self.runJob("xmipp_metadata_utilities","-i %s.xmd --fill anglePsi rand_uniform 0 360 "%(fnRoot))
# If there is an initial volume, assign angles
if self.initialVolume.hasValue():
fnGallery=self._getTmpPath('gallery_InitialVolume.stk')
self.runJob("xmipp_angular_projection_matching", "-i %s.xmd -o %s.xmd --ref %s --Ri 0 --Ro %s --max_shift %s --append"\
%(fnRoot,fnRoot,fnGallery,str(self.Xdim/2),str(self.Xdim/20)))
# Reconstruct with the small sample
self.reconstructStep(fnRoot)
fnVol = fnRoot+'.vol'
# Generate projections from this reconstruction
fnGallery=self._getTmpPath('gallery_'+fnBase+'.stk')
self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\
%(fnVol,fnGallery,self.angularSampling.get(),self.symmetryGroup.get(),fnOutputReducedClass))
# Assign angles to the rest of images
fnAngles=self._getTmpPath('angles_'+fnBase+'.xmd')
self.runJob("xmipp_angular_projection_matching", "-i %s -o %s --ref %s --Ri 0 --Ro %s --max_shift %s --append"\
%(fnOutputReducedClass,fnAngles,fnGallery,str(self.Xdim/2),str(self.Xdim/20)))
# Delete intermediate files
cleanPath(fnGallery)
cleanPath(self._getTmpPath('gallery_'+fnBase+'_sampling.xmd'))
cleanPath(self._getTmpPath('gallery_'+fnBase+'.doc'))
cleanPath(fnVol)
cleanPath(self._getTmpPath(fnBase+'.xmd'))
if self.initialVolume.hasValue():
cleanPattern(self._getTmpPath("gallery_InitialVolume*"))
def createOutputStep(self):
# PARTICLES
cleanPattern(self._getPath("*.sqlite"))
partSet = self._createSetOfParticles()
readSetOfParticles(self._getPath("particles.xmd"), partSet)
inputSampling = self.inputCoordinates[0].get().getMicrographs(
).getSamplingRate()
partSet.setSamplingRate(self._getDownFactor() * inputSampling)
boxSize = self._getBoxSize()
# COORDINATES
writeSet = False
if self.checkIfPrevRunIsCompatible("coords_"):
writeSet = True
if not "OR" in self.coordinatesDict:
self.loadCoords(self._getExtraPath(
self.CONSENSUS_COOR_PATH_TEMPLATE % 'TRUE'),
'OR',
writeSet=False)
coordSet = SetOfCoordinates(
filename=self._getPath("coordinates.sqlite"))
coordSet.copyInfo(self.coordinatesDict['OR'])
coordSet.setBoxSize(boxSize)
coordSet.setMicrographs(self.coordinatesDict['OR'].getMicrographs())
downFactor = self._getDownFactor()
for part in partSet:
coord = part.getCoordinate().clone()
coord.scale(downFactor)
deepZscoreLabel = '_xmipp_%s' % xmipp.label2Str(
MD.MDL_ZSCORE_DEEPLEARNING1)
setattr(coord, deepZscoreLabel, getattr(part, deepZscoreLabel))
coordSet.append(coord)
coordSet.write()
partSet.write()
self._defineOutputs(outputCoordinates=coordSet)
self._defineOutputs(outputParticles=partSet)
for inSetOfCoords in self.inputCoordinates:
self._defineSourceRelation(inSetOfCoords.get(), coordSet)
self._defineSourceRelation(inSetOfCoords.get(), partSet)
def mergeStep(self, iterN):
jobs, _ = self._getJobsParams()
self._mergeAllParFiles(iterN, jobs)
argsStr = self._getMergeArgs()
paramsDic = {
'output_cls': self._getFileName('iter_cls', iter=iterN),
'dumpSeed': self._getFileName('iter_cls_block_seed', iter=iterN),
'numberOfJobs': jobs
}
cmdArgs = argsStr % paramsDic
self.runJob(self._getProgram('merge2d'),
cmdArgs,
cwd=self._getExtraPath(),
env=Plugin.getEnviron())
dumpFns = self._getExtraPath(
'Refine2D/ClassAverages/class_dump_file_%d_*' % iterN)
cleanPattern(dumpFns)
def cleanPrime(self):
self._enterDir(self._getExtraPath())
cleanPath("cmdline.txt")
cleanPattern("*.txt")
cleanPattern("startvol_state*.spi")
# Get last iteration
for i in range(1, self.getLastIteration()):
cleanPattern("recvol_state*_iter%d.spi" % i)
self._leaveDir()
def projectStep(self, numProj, iniRot, endRot, iniTilt, endTilt, fn, idx):
newXdim = readInfoField(self._getExtraPath(), "size",
emlib.MDL_XSIZE)
fnVol = self._getTmpPath(self._fnVolumeVol)
uniformProjectionsStr = """
# XMIPP_STAR_1 *
data_block1
_dimensions2D '%d %d'
_projRotRange '%d %d %d'
_projRotRandomness random
_projRotNoise '0'
_projTiltRange '%d %d 1'
_projTiltRandomness random
_projTiltNoise '0'
_projPsiRange '0 0 1'
_projPsiRandomness random
_projPsiNoise '0'
_noisePixelLevel '0'
_noiseCoord '0'
""" % (newXdim, newXdim, iniRot, endRot, numProj, iniTilt, endTilt)
fnParams = self._getExtraPath("uniformProjections%d.xmd" % idx)
fh = open(fnParams, "w")
fh.write(uniformProjectionsStr)
fh.close()
fnProjs = self._getExtraPath(fn + "%d.stk" % idx)
self.runJob("xmipp_phantom_project",
"-i %s -o %s --method fourier 1 0.5 "
"--params %s" % (fnVol, fnProjs, fnParams), numberOfMpi=1)
fnProjsXmd=fnProjs[:-3]+'xmd'
self.runJob("xmipp_metadata_utilities",
"-i %s --fill ref lineal 1 1 " % (fnProjsXmd), numberOfMpi=1)
cleanPattern(self._getExtraPath('uniformProjections*'))
def cleanPrime(self):
self._enterDir(self._getExtraPath())
cleanPath("cmdline.txt")
cleanPattern("*.txt")
cleanPattern("startvol_state*.spi")
# Get last iteration
for i in range(1, self.getLastIteration()):
cleanPattern("recvol_state*_iter%d.spi"%i)
self._leaveDir()
def refineAnglesStep(self):
fnTmpDir = self._getTmpPath()
fnDirectional = self._getDirectionalClassesFn()
inputParticles = self.inputParticles.get()
newTs = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE)
# Generate projections
fnGallery = join(fnTmpDir, "gallery.stk")
fnGalleryMd = join(fnTmpDir, "gallery.doc")
fnVol = self._getInputVolFn()
args = "-i %s -o %s --sampling_rate %f --sym %s" % \
(fnVol, fnGallery, 5.0, self.symmetryGroup)
args += " --compute_neighbors --angular_distance -1 --experimental_images %s" % fnDirectional
self.runJob("xmipp_angular_project_library",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
# Global angular assignment
maxShift = 0.15 * newXdim
fnAngles = join(fnTmpDir, "angles_iter001_00.xmd")
if not self.useGpu.get():
args = '-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0' % \
(fnDirectional, fnGalleryMd, maxShift, fnTmpDir)
self.runJob('xmipp_reconstruct_significant',
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
else:
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
args = '-i %s -r %s -o %s --dev %s ' % (fnDirectional, fnGalleryMd,
fnAngles, GpuListCuda)
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --operate drop_column ref" % fnAngles,
numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set join %s ref2" % (fnAngles, fnDirectional),
numberOfMpi=1)
# Local angular assignment
fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
args = "-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 " % \
(fnAngles, fnAnglesLocalStk, newTs, newXdim / 2, 2, fnVol,
self.targetResolution)
args += " --optimizeShift --max_shift %f" % maxShift
args += " --optimizeAngles --max_angular_change %f" % self.angularDistance
self.runJob("xmipp_angular_continuous_assign2",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
moveFile(self._getPath("directional_local_classes.xmd"),
self._getDirectionalClassesFn())
cleanPattern(self._getExtraPath("direction_*"))
def managingOutputFilesStep(self):
volList = [vol.clone() for vol in self._iterInputVolumes()]
copyFile(self._getExtraPath("CoordinateMatrixColumnF1.txt"),
self._defineResultsName1())
copyFile(self._getExtraPath("CoordinateMatrixColumnF2.txt"),
self._defineResultsName2())
copyFile(self._getExtraPath("CoordinateMatrixColumnF3.txt"),
self._defineResultsName3())
cleanPattern(self._getExtraPath('pseudoatoms*'))
cleanPattern(self._getExtraPath('vec_ani.pkl'))
cleanPattern(self._getExtraPath('CoordinateMatrixColumnF*'))
cleanPattern(self._getPath('modes/vec*'))
cleanPath(self._getPath('modes'))
if not self.keepingOutputFiles.get():
cleanPattern(self._getPath('warnings*'))
cleanPattern(self._getPath('outputRigid*'))
cleanPattern(self._getExtraPath('RigidAlign*'))
cleanPattern(self._getExtraPath('Pseudoatoms*'))
cleanPattern(self._getExtraPath('comp*'))
cleanPattern(self._getExtraPath('Deform*'))
def refineStep(self, args):
""" Run the EMAN program to refine a volume. """
if not self.doContinue:
cleanPattern(self._getExtraPath('refine_01'))
program = getEmanProgram('e2refine_easy.py')
self.runJob(program, args, cwd=self._getExtraPath())
def managingOutputFilesStep(self):
cleanPattern(self._getPath('pseudoatoms*'))
cleanPattern(self._getPath('modes'))
cleanPattern(self._getExtraPath('vec_ani.pkl'))
if not self.keepingOutputFiles.get():
cleanPattern(self._getPath('warnings*'))
cleanPattern(self._getPath('outputRigid*'))
cleanPattern(self._getPath('modes*.xmd'))
cleanPattern(self._getExtraPath('RigidAlign*'))
cleanPattern(self._getExtraPath('pseudoatoms*'))
cleanPattern(self._getExtraPath('Pseudoatoms*'))
cleanPattern(self._getExtraPath('comp*'))
cleanPattern(self._getExtraPath('Deform*'))
cleanPattern(self._getExtraPath('transformation-matrix*'))
cleanPattern(self._getExtraPath('modes*'))
def managingOutputFilesStep(self):
volList = [vol.clone() for vol in self._iterInputVolumes()]
copyFile (self._getExtraPath ("CoordinateMatrixColumnF1.txt"),
self._defineResultsName1())
copyFile (self._getExtraPath ("CoordinateMatrixColumnF2.txt"),
self._defineResultsName2())
copyFile (self._getExtraPath ("CoordinateMatrixColumnF3.txt"),
self._defineResultsName3())
cleanPattern(self._getExtraPath('pseudoatoms*'))
cleanPattern(self._getExtraPath('vec_ani.pkl'))
cleanPattern(self._getExtraPath('CoordinateMatrixColumnF*'))
cleanPattern(self._getPath('modes/vec*'))
cleanPath(self._getPath('modes'))
if not self.keepingOutputFiles.get():
cleanPattern(self._getPath('warnings*'))
cleanPattern(self._getPath('outputRigid*'))
cleanPattern(self._getExtraPath('RigidAlign*'))
cleanPattern(self._getExtraPath('Pseudoatoms*'))
cleanPattern(self._getExtraPath('comp*'))
cleanPattern(self._getExtraPath('Deform*'))
def _generateAnimation(self):
prot = self.protocol
projectorFile = prot.getProjectorFile()
animation = self.trajectoriesWindow.getAnimationName()
animationPath = prot._getExtraPath('animation_%s' % animation)
cleanPath(animationPath)
makePath(animationPath)
animationRoot = join(animationPath, 'animation_%s' % animation)
trajectoryPoints = np.array(
[p.getData() for p in self.trajectoriesWindow.pathData])
if projectorFile:
M = np.loadtxt(projectorFile)
deformations = np.dot(trajectoryPoints, np.linalg.pinv(M))
np.savetxt(animationRoot + 'trajectory.txt', trajectoryPoints)
else:
Y = np.loadtxt(prot.getOutputMatrixFile())
X = np.loadtxt(prot.getDeformationFile())
# Find closest points in deformations
deformations = [
X[np.argmin(np.sum((Y - p)**2, axis=1))]
for p in trajectoryPoints
]
pdb = prot.getInputPdb()
pdbFile = pdb.getFileName()
modesFn = prot.inputNMA.get()._getExtraPath('modes.xmd')
for i, d in enumerate(deformations):
atomsFn = animationRoot + 'atomsDeformed_%02d.pdb' % (i + 1)
cmd = '-o %s --pdb %s --nma %s --deformations %s' % (
atomsFn, pdbFile, modesFn, str(d)[1:-1])
runJob(None, 'xmipp_pdb_nma_deform', cmd, env=prot._getEnviron())
# Join all deformations in a single pdb
# iterating going up and down through all points
# 1 2 3 ... n-2 n-1 n n-1 n-2 ... 3, 2
n = len(deformations)
r1 = range(1, n + 1)
r2 = range(2, n) # Skip 1 at the end
r2.reverse()
loop = r1 + r2
trajFn = animationRoot + '.pdb'
trajFile = open(trajFn, 'w')
for i in loop:
atomsFn = animationRoot + 'atomsDeformed_%02d.pdb' % i
atomsFile = open(atomsFn)
for line in atomsFile:
trajFile.write(line)
trajFile.write('TER\nENDMDL\n')
atomsFile.close()
trajFile.close()
# Delete temporary atom files
cleanPattern(animationRoot + 'atomsDeformed_??.pdb')
# Generate the vmd script
vmdFn = animationRoot + '.vmd'
vmdFile = open(vmdFn, 'w')
vmdFile.write("""
mol new %s
animate style Loop
display projection Orthographic
mol modcolor 0 0 Index
mol modstyle 0 0 Beads 1.000000 8.000000
animate speed 0.5
animate forward
""" % trajFn)
vmdFile.close()
VmdView(' -e ' + vmdFn).show()
def reconstructionStep(self, numberOfImages, fractionCounter,
iteration, debugging,
fnNewImgMd, particlesMd):
fnRoot = self._getExtraPath("fraction%02d"%fractionCounter)
Ts = self.inputParticles.get().getSamplingRate()
#for noise
fnRootN = self._getExtraPath("Nfraction%02d"%fractionCounter)
for i in range(0,2):
fnImgs = fnRoot+"_images_%02d_%02d.xmd"%(i, iteration)
if self.doResize.get():
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" % (
fnNewImgMd, fnImgs,numberOfImages),
numberOfMpi = 1)
else:
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" % (
particlesMd,fnImgs,numberOfImages),
numberOfMpi = 1)
params = ' -i %s' % fnImgs
params += ' -o %s' % fnRoot+"_%02d_%02d.vol" % (i, iteration)
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --max_resolution %0.3f' % self.maxRes
params += ' --padding 2'
params += ' --thr 1'
#params += ' --thr %d' % self.numberOfThreads.get()
params += ' --sampling %f' % Ts
self.runJob('xmipp_reconstruct_fourier', params)
#for noise
noiseStk = fnRoot+"_noises_%02d.stk"%i
self.runJob ("xmipp_image_convert",
"-i %s -o %s" % (fnImgs, noiseStk), numberOfMpi = 1)
self.runJob("xmipp_image_operate",
"-i %s --mult 0" % noiseStk)
self.runJob("xmipp_transform_add_noise",
"-i %s --type gaussian 3" % noiseStk, numberOfMpi = 1)
fnImgsNL = fnRoot+"_noisesL_%02d.xmd" % i
noiseStk2 = fnRoot+"_noises2_%02d.stk" % i
self.runJob ("xmipp_image_convert",
"-i %s -o %s --save_metadata_stack %s" % (
noiseStk, noiseStk2, fnImgsNL), numberOfMpi = 1)
fnImgsNoiseOld = fnRoot+"_noisesOld_%02d.xmd" % i
fnImgsN = fnRoot+"_noises_%02d_%02d.xmd" % (i, iteration)
self.runJob("xmipp_metadata_utilities",
'-i %s -o %s --operate drop_column "image"' % (
fnImgs,fnImgsNoiseOld), numberOfMpi = 1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set merge %s -o %s" % (
fnImgsNL, fnImgsNoiseOld, fnImgsN), numberOfMpi = 1)
#alignment gaussian noise
fnImgsAlign = self._getExtraPath("Nfraction_alignment%02d" % fractionCounter)
fnImgsAlignN = fnImgsAlign + "_%02d_%02d.xmd" % (i,
iteration)
args="-i %s -o %s -r %s --Ri 0 --Ro -1 --mem 2 --append " % (
fnImgsN,fnImgsAlignN,
self._getExtraPath("Ref_Projections.stk"))
self.runJob('xmipp_angular_projection_matching',
args,
numberOfMpi = self.numberOfMpi.get())
#numberOfMpi = self.numberOfMpi.get() * self.numberOfThreads.get())
params = ' -i %s' % fnImgsAlignN
params += ' -o %s' % fnRootN+"_%02d_%02d.vol"%(i, iteration)
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --max_resolution %0.3f' % self.maxRes
params += ' --padding 2'
params += ' --thr 1'
#params += ' --thr %d' % self.numberOfThreads.get()
params += ' --sampling %f' % Ts
self.runJob('xmipp_reconstruct_fourier', params)
self.runJob('xmipp_resolution_fsc',
"--ref %s -i %s -o %s --sampling_rate %f" % \
(fnRoot + "_00_%02d.vol" % iteration,
fnRoot + "_01_%02d.vol" % iteration,
fnRoot + "_fsc_%02d.xmd" % iteration,Ts),
numberOfMpi = 1)
mdFSC = xmipp.MetaData(fnRoot + "_fsc_%02d.xmd" % iteration)
for id in mdFSC:
fscValue = mdFSC.getValue(xmipp.MDL_RESOLUTION_FRC,id)
maxFreq = mdFSC.getValue(xmipp.MDL_RESOLUTION_FREQREAL,id)
if fscValue < 0.5:
break
fh = open(fnRoot + "_freq.txt","a")
fh.write("%f\n" % maxFreq)
fh.close()
#for noise
self.runJob('xmipp_resolution_fsc',
"--ref %s -i %s -o %s --sampling_rate %f" % (
fnRootN + "_00_%02d.vol" % iteration,
fnRootN + "_01_%02d.vol" % iteration,
fnRootN + "_fsc_%02d.xmd" % iteration,Ts),
numberOfMpi = 1)
cleanPattern(fnRoot + "_noises_0?_0?.xmd")
cleanPattern(fnRoot + "_noisesOld_0?.xmd")
cleanPattern(fnRoot + "_noisesL_0?.xmd")
cleanPattern(fnRoot + "_noises2_0?.stk")
mdFSCN = xmipp.MetaData(fnRootN + "_fsc_%02d.xmd" % iteration)
for id in mdFSCN:
fscValueN = mdFSCN.getValue(xmipp.MDL_RESOLUTION_FRC, id)
maxFreqN = mdFSCN.getValue(xmipp.MDL_RESOLUTION_FREQREAL, id)
if fscValueN < 0.5:
break
fhN = open(fnRootN + "_freq.txt", "a")
fhN.write("%f\n" % maxFreqN)
fhN.close()
if not debugging:
cleanPattern(fnRoot + "_0?_0?.vol")
cleanPattern(fnRoot + "_images_0?_0?.xmd")
cleanPattern(fnRoot + "_fsc_0?.xmd")
cleanPattern(fnRootN + "_0?_0?.vol")
cleanPattern(fnRoot + "_noises_0?.stk")
cleanPattern(fnRootN + "_fsc_0?.xmd")
cleanPattern(fnImgsAlign + "_0?_0?.xmd")
def reconstructVolumeStep(self, args):
""" Run the EMAN program to reconstruct a volume. """
cleanPattern(self._getFileName("volume"))
program = getEmanProgram('e2make3d.py')
self.runJob(program, args, cwd=self._getExtraPath())
def createOutputStep(self):
inputParticles = self.inputParticles.get()
# if not self._useSeveralClasses():
# newTs = inputParticles.getSamplingRate()
# else:
# newTs = self.readInfoField(self._getExtraPath(), "sampling",
# xmipp.MDL_SAMPLINGRATE)
self.mdClasses = xmippLib.MetaData(self._getDirectionalClassesFn())
self.mdImages = xmippLib.MetaData(self._getDirectionalImagesFn())
origTs = inputParticles.getSamplingRate()
lastTs = self.readInfoField(self._getExtraPath(), "sampling",
xmippLib.MDL_SAMPLINGRATE)
if origTs!=lastTs:
newXdim=inputParticles.getXDim()
self.runJob("xmipp_image_resize", "-i %s -o %s --save_metadata_stack %s --fourier %d"
% (self._getDirectionalClassesFn(),
self._getPath("aux_directional_local_classes.stk"),
self._getPath("aux_directional_classes.xmd"),
newXdim), numberOfMpi=1)
from shutil import copy
copy(self._getPath("aux_directional_local_classes.stk"),
self._getPath("directional_local_classes.stk"))
copy(self._getPath("aux_directional_classes.xmd"),
self._getPath("directional_classes.xmd"))
cleanPattern(self._getPath("aux_directional*"))
classes2D = self._createSetOfClasses2D(inputParticles)
#classes2D.getImages().setSamplingRate(newTs)
classes2D.getImages().setSamplingRate(origTs)
self.averageSet = self._createSetOfAverages()
self.averageSet.copyInfo(inputParticles)
self.averageSet.setAlignmentProj()
#self.averageSet.setSamplingRate(newTs)
self.averageSet.setSamplingRate(origTs)
# Let's use a SetMdIterator because it should be less particles
# in the metadata produced than in the input set
iterator = md.SetMdIterator(self.mdImages, sortByLabel=md.MDL_ITEM_ID,
updateItemCallback=self._updateParticle,
skipDisabled=True)
fnHomogeneous = self._getExtraPath("images_homogeneous.xmd")
if exists(fnHomogeneous):
if origTs != lastTs:
newXdim = inputParticles.getXDim()
self.runJob("xmipp_image_resize", "-i %s --dim %d"
% (fnHomogeneous, newXdim), numberOfMpi=1)
homogeneousSet = self._createSetOfParticles()
homogeneousSet.copyInfo(inputParticles)
#homogeneousSet.getImages().setSamplingRate(newTs)
homogeneousSet.getImages().setSamplingRate(origTs)
homogeneousSet.setAlignmentProj()
self.iterMd = md.iterRows(fnHomogeneous, md.MDL_PARTICLE_ID)
self.lastRow = next(self.iterMd)
homogeneousSet.copyItems(inputParticles,
updateItemCallback=self._updateHomogeneousItem)
self._defineOutputs(outputHomogeneous=homogeneousSet)
self._defineSourceRelation(self.inputParticles, homogeneousSet)
#AJ testing
#AJ por que desaparece una clase que tiene imagenes asignadas
listRefId=[]
for row in md.iterRows(self.mdClasses, xmippLib.MDL_REF2):
refId = row.getValue(xmippLib.MDL_REF2, row.getObjId())
if len(listRefId)>0 and refId != listRefId[-1]+1:
whereEnd = listRefId[-1]+1
for i in range(refId-whereEnd):
rowNew = row
rowNew.setValue(xmippLib.MDL_REF2, listRefId[-1]+i+1)
rowNew.setValue(xmippLib.MDL_IMAGE, 'None')
rowNew.setValue(xmippLib.MDL_IMAGE1, 'None')
rowNew.addToMd(self.mdClasses)
listRefId.append(listRefId[-1]+i+1)
listRefId.append(refId)
else:
listRefId.append(refId)
self.mdClasses.write(self._getDirectionalClassesFn())
self.mdClasses = xmippLib.MetaData(self._getDirectionalClassesFn())
#END AJ
classes2D.classifyItems(updateItemCallback=iterator.updateItem,
updateClassCallback=self._updateClass)
self._defineOutputs(outputClasses=classes2D)
self._defineOutputs(outputAverages=self.averageSet)
self._defineSourceRelation(self.inputParticles, classes2D)
self._defineSourceRelation(self.inputParticles, self.averageSet)
if self.splitVolume and self.directionalClasses.get() > 1:
volumesSet = self._createSetOfVolumes()
#volumesSet.setSamplingRate(newTs)
volumesSet.setSamplingRate(origTs)
for i in range(2):
vol = Volume()
if origTs != lastTs:
newXdim = inputParticles.getXDim()
self.runJob("xmipp_image_resize", "-i %s --dim %d"
% (self._getExtraPath("split%d.vol" % (i + 1)),
newXdim), numberOfMpi=1)
vol.setLocation(1, self._getExtraPath("split%d.vol" % (i + 1)))
volumesSet.append(vol)
self._defineOutputs(outputVolumes=volumesSet)
self._defineSourceRelation(inputParticles, volumesSet)
def gatherResultsStep(self, debugging):
self._writeFreqsMetaData("fraction*_freq.txt",
self._defineResultsName())
self._writeFreqsMetaData("Nfraction*_freq.txt",
self._defineResultsNoiseName())
if not debugging:
cleanPattern(self._getExtraPath("fraction*_freq.txt"))
cleanPattern(self._getExtraPath("Nfraction*_freq.txt"))
cleanPattern(self._getExtraPath('Ref_Projections*'))
cleanPattern(self._getExtraPath('newImages.stk'))
cleanPattern(self._getExtraPath('newImages.xmd'))
cleanPattern(self._getExtraPath('newVolume.vol'))
def cleaningStep(self):
cleanPattern(self._getExtraPath("gallery*"))
cleanPattern(self._getExtraPath("mask.vol"))
cleanPattern(self._getExtraPath("neighbours.xmd"))
cleanPattern(self._getExtraPath("split00*"))
def reconstructionStep(self, numberOfImages, fractionCounter, iteration,
debugging, fnNewImgMd, particlesMd):
fnRoot = self._getExtraPath("fraction%02d" % fractionCounter)
Ts = self.inputParticles.get().getSamplingRate()
#for noise
fnRootN = self._getExtraPath("Nfraction%02d" % fractionCounter)
for i in range(0, 2):
fnImgs = fnRoot + "_images_%02d_%02d.xmd" % (i, iteration)
if self.doResize.get():
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" %
(fnNewImgMd, fnImgs, numberOfImages),
numberOfMpi=1)
else:
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" %
(particlesMd, fnImgs, numberOfImages),
numberOfMpi=1)
params = ' -i %s' % fnImgs
params += ' -o %s' % fnRoot + "_%02d_%02d.vol" % (i, iteration)
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --max_resolution %0.3f' % self.maxRes
params += ' --padding 2'
params += ' --thr %d' % 1 # self.numberOfThreads.get()
params += ' --sampling %f' % Ts
self.runJob('xmipp_reconstruct_fourier', params)
#for noise
noiseStk = fnRoot + "_noises_%02d.stk" % i
self.runJob("xmipp_image_convert",
"-i %s -o %s" % (fnImgs, noiseStk),
numberOfMpi=1)
self.runJob("xmipp_image_operate", "-i %s --mult 0" % noiseStk)
self.runJob("xmipp_transform_add_noise",
"-i %s --type gaussian 3" % noiseStk,
numberOfMpi=1)
fnImgsNL = fnRoot + "_noisesL_%02d.xmd" % i
noiseStk2 = fnRoot + "_noises2_%02d.stk" % i
self.runJob("xmipp_image_convert",
"-i %s -o %s --save_metadata_stack %s" %
(noiseStk, noiseStk2, fnImgsNL),
numberOfMpi=1)
fnImgsNoiseOld = fnRoot + "_noisesOld_%02d.xmd" % i
fnImgsN = fnRoot + "_noises_%02d_%02d.xmd" % (i, iteration)
self.runJob("xmipp_metadata_utilities",
'-i %s -o %s --operate drop_column "image"' %
(fnImgs, fnImgsNoiseOld),
numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set merge %s -o %s" %
(fnImgsNL, fnImgsNoiseOld, fnImgsN),
numberOfMpi=1)
#alignment gaussian noise
fnImgsAlign = self._getExtraPath("Nfraction_alignment%02d" %
fractionCounter)
fnImgsAlignN = fnImgsAlign + "_%02d_%02d.xmd" % (i, iteration)
args = "-i %s -o %s -r %s --Ri 0 --Ro -1 --mem 2 --append " % (
fnImgsN, fnImgsAlignN,
self._getExtraPath("Ref_Projections.stk"))
self.runJob('xmipp_angular_projection_matching',
args,
numberOfMpi=self.numberOfMpi.get()
) # * self.numberOfThreads.get())
params = ' -i %s' % fnImgsAlignN
params += ' -o %s' % fnRootN + "_%02d_%02d.vol" % (i, iteration)
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --max_resolution %0.3f' % self.maxRes
params += ' --padding 2'
params += ' --thr %d' % 1 # self.numberOfThreads.get()
params += ' --sampling %f' % Ts
self.runJob('xmipp_reconstruct_fourier', params)
self.runJob('xmipp_resolution_fsc',
"--ref %s -i %s -o %s --sampling_rate %f" % \
(fnRoot + "_00_%02d.vol" % iteration,
fnRoot + "_01_%02d.vol" % iteration,
fnRoot + "_fsc_%02d.xmd" % iteration,Ts),
numberOfMpi = 1)
mdFSC = xmipp.MetaData(fnRoot + "_fsc_%02d.xmd" % iteration)
for id in mdFSC:
fscValue = mdFSC.getValue(xmipp.MDL_RESOLUTION_FRC, id)
maxFreq = mdFSC.getValue(xmipp.MDL_RESOLUTION_FREQREAL, id)
if fscValue < 0.5:
break
fh = open(fnRoot + "_freq.txt", "a")
fh.write("%f\n" % maxFreq)
fh.close()
#for noise
self.runJob('xmipp_resolution_fsc',
"--ref %s -i %s -o %s --sampling_rate %f" %
(fnRootN + "_00_%02d.vol" % iteration,
fnRootN + "_01_%02d.vol" % iteration,
fnRootN + "_fsc_%02d.xmd" % iteration, Ts),
numberOfMpi=1)
cleanPattern(fnRoot + "_noises_0?_0?.xmd")
cleanPattern(fnRoot + "_noisesOld_0?.xmd")
cleanPattern(fnRoot + "_noisesL_0?.xmd")
cleanPattern(fnRoot + "_noises2_0?.stk")
mdFSCN = xmipp.MetaData(fnRootN + "_fsc_%02d.xmd" % iteration)
for id in mdFSCN:
fscValueN = mdFSCN.getValue(xmipp.MDL_RESOLUTION_FRC, id)
maxFreqN = mdFSCN.getValue(xmipp.MDL_RESOLUTION_FREQREAL, id)
if fscValueN < 0.5:
break
fhN = open(fnRootN + "_freq.txt", "a")
fhN.write("%f\n" % maxFreqN)
fhN.close()
if not debugging:
cleanPattern(fnRoot + "_0?_0?.vol")
cleanPattern(fnRoot + "_images_0?_0?.xmd")
cleanPattern(fnRoot + "_fsc_0?.xmd")
cleanPattern(fnRootN + "_0?_0?.vol")
cleanPattern(fnRoot + "_noises_0?.stk")
cleanPattern(fnRootN + "_fsc_0?.xmd")
cleanPattern(fnImgsAlign + "_0?_0?.xmd")
def createInitialModelStep(self, args):
""" Run the EMAN program to create the initial model. """
cleanPattern(self._getExtraPath('initial_models'))
program = eman2.getEmanProgram('e2initialmodel.py')
self.runJob(program, args, cwd=self._getExtraPath())
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
for i, vol in enumerate(self._iterInputVols()):
volDir = self._getVolDir(i + 1)
volume = vol.clone()
volPrefix = 'vol%03d_' % (i + 1)
m_pruned = md.MetaData()
m_pruned.read(volDir + '/pruned_particles_alignability.xmd')
prunedMd = self._getExtraPath(
volPrefix + 'pruned_particles_alignability.xmd')
moveFile(join(volDir, 'pruned_particles_alignability.xmd'),
prunedMd)
m_volScore = md.MetaData()
m_volScore.read(volDir + '/validationAlignability.xmd')
validationMd = self._getExtraPath(
volPrefix + 'validation_alignability.xmd')
moveFile(join(volDir, 'validationAlignability.xmd'), validationMd)
imgSet = self.inputParticles.get()
outImgSet = self._createSetOfParticles(volPrefix)
outImgSet.copyInfo(imgSet)
outImgSet.copyItems(imgSet,
updateItemCallback=self._setWeight,
itemDataIterator=md.iterRows(prunedMd,
sortByLabel=md.MDL_ITEM_ID))
mdValidatoin = md.getFirstRow(validationMd)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_ALIGNABILITY)
volume.weightAlignabilityPrecision = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_ACCURACY_ALIGNABILITY)
volume.weightAlignabilityAccuracy = Float(weight)
weight = mdValidatoin.getValue(md.MDL_WEIGHT_PRECISION_MIRROR)
volume.weightMirror = Float(weight)
volume.cleanObjId() # clean objects id to assign new ones inside the set
outputVols.append(volume)
self._defineOutputs(outputParticles=outImgSet)
self.createPlot2D(volPrefix, m_pruned)
outputVols.setSamplingRate(volume.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
cleanPattern(self._getPath("reference_particles.*"))
cleanPattern(self._getExtraPath("scaled_particles.*"))
cleanPattern(self._getExtraPath("reference_particles.*"))
cleanPattern(self._getExtraPath("corrected_ctf_particles.*"))
cleanPattern(self._getFileName("volume"))
cleanPattern(self._getExtraPath("params.txt"))
def _generateAnimation(self):
prot = self.protocol
projectorFile = prot.getProjectorFile()
animation = self.trajectoriesWindow.getAnimationName()
animationPath = prot._getExtraPath('animation_%s' % animation)
cleanPath(animationPath)
makePath(animationPath)
animationRoot = join(animationPath, 'animation_%s' % animation)
trajectoryPoints = np.array([p.getData() for p in self.trajectoriesWindow.pathData])
if projectorFile:
M = np.loadtxt(projectorFile)
deformations = np.dot(trajectoryPoints, np.linalg.pinv(M))
np.savetxt(animationRoot + 'trajectory.txt', trajectoryPoints)
else:
Y = np.loadtxt(prot.getOutputMatrixFile())
X = np.loadtxt(prot.getDeformationFile())
# Find closest points in deformations
deformations = [X[np.argmin(np.sum((Y - p)**2, axis=1))] for p in trajectoryPoints]
pdb = prot.getInputPdb()
pdbFile = pdb.getFileName()
modesFn = prot.inputNMA.get()._getExtraPath('modes.xmd')
for i, d in enumerate(deformations):
atomsFn = animationRoot + 'atomsDeformed_%02d.pdb' % (i+1)
cmd = '-o %s --pdb %s --nma %s --deformations %s' % (atomsFn, pdbFile, modesFn, str(d)[1:-1])
runJob(None, 'xmipp_pdb_nma_deform', cmd, env=prot._getEnviron())
# Join all deformations in a single pdb
# iterating going up and down through all points
# 1 2 3 ... n-2 n-1 n n-1 n-2 ... 3, 2
n = len(deformations)
r1 = range(1, n+1)
r2 = range(2, n) # Skip 1 at the end
r2.reverse()
loop = r1 + r2
trajFn = animationRoot + '.pdb'
trajFile = open(trajFn, 'w')
for i in loop:
atomsFn = animationRoot + 'atomsDeformed_%02d.pdb' % i
atomsFile = open(atomsFn)
for line in atomsFile:
trajFile.write(line)
trajFile.write('TER\nENDMDL\n')
atomsFile.close()
trajFile.close()
# Delete temporary atom files
cleanPattern(animationRoot + 'atomsDeformed_??.pdb')
# Generate the vmd script
vmdFn = animationRoot + '.vmd'
vmdFile = open(vmdFn, 'w')
vmdFile.write("""
mol new %s
animate style Loop
display projection Orthographic
mol modcolor 0 0 Index
mol modstyle 0 0 Beads 1.000000 8.000000
animate speed 0.5
animate forward
""" % trajFn)
vmdFile.close()
VmdView(' -e ' + vmdFn).show()
def refineStep(self, args):
""" Run the EMAN program to refine a volume. """
if not self.doContinue:
cleanPattern(self._getExtraPath('refine_01'))
program = getEmanProgram('e2refine_easy.py')
self.runJob(program, args, cwd=self._getExtraPath())
def significantStep(self, iterNumber, alpha):
iterDir = self._getTmpPath('iter%03d' % iterNumber)
makePath(iterDir)
prevVolFn = self.getIterVolume(iterNumber - 1)
volFn = self.getIterVolume(iterNumber)
anglesFn = self._getExtraPath('angles_iter%03d.xmd' % iterNumber)
t = Timer()
t.tic()
if self.useGPU and iterNumber > 1:
# Generate projections
fnGalleryRoot = join(iterDir, "gallery")
args = "-i %s -o %s.stk --sampling_rate %f --sym %s " \
"--compute_neighbors --angular_distance -1 " \
"--experimental_images %s --min_tilt_angle %f " \
"--max_tilt_angle %f -v 0 --perturb %f" % \
(prevVolFn, fnGalleryRoot, self.angularSampling,
self.symmetryGroup, self.imgsFn, self.minTilt, self.maxTilt,
math.sin(self.angularSampling.get()) / 4)
self.runJob("xmipp_angular_project_library ", args)
# Align
# TODO check the alpha values for gpu
if self.trueSymsNo != 0:
alphaApply = (alpha * self.trueSymsNo) / 2
else:
alphaApply = alpha / 2
if self.maximumShift == -1:
maxShift = 10
else:
maxShift = self.maximumShift
args = '-i_ref %s.doc -i_exp %s -o %s --significance %f ' \
'--maxShift %f' % \
(fnGalleryRoot, self.imgsFn, anglesFn, alphaApply,
maxShift)
self.runJob("xmipp_cuda_correlation", args, numberOfMpi=1)
cleanPattern(fnGalleryRoot + "*")
else:
args = self.getSignificantArgs(self.imgsFn)
args += ' --odir %s' % iterDir
args += ' --alpha0 %f --alphaF %f' % (alpha, alpha)
if iterNumber == 1:
if self.thereisRefVolume:
args += " --initvolumes " + \
self._getExtraPath('input_volumes.xmd')
else:
args += " --numberOfVolumes 1"
else:
args += " --initvolumes %s" % prevVolFn
self.runJob("xmipp_reconstruct_significant", args)
moveFile(os.path.join(iterDir, 'angles_iter001_00.xmd'), anglesFn)
t.toc('Significant took: ')
reconsArgs = ' -i %s' % anglesFn
reconsArgs += ' -o %s' % volFn
reconsArgs += ' --weight -v 0 --sym %s ' % self.symmetryGroup
print "Number of images for reconstruction: ", metadata.getSize(
anglesFn)
t.tic()
self.runJob("xmipp_reconstruct_fourier", reconsArgs)
t.toc('Reconstruct fourier took: ')
# Center the volume
fnSym = self._getExtraPath('volumeSym_%03d.vol' % iterNumber)
self.runJob("xmipp_transform_mirror",
"-i %s -o %s --flipX" % (volFn, fnSym),
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipY" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipZ" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_image_operate",
"-i %s --plus %s" % (fnSym, volFn),
numberOfMpi=1)
self.runJob("xmipp_volume_align",
'--i1 %s --i2 %s --local --apply' % (fnSym, volFn),
numberOfMpi=1)
cleanPath(fnSym)
# To mask the volume
xdim = self.inputSet.get().getDim()[0]
maskArgs = "-i %s --mask circular %d -v 0" % (volFn, -xdim / 2)
self.runJob('xmipp_transform_mask', maskArgs, numberOfMpi=1)
# TODO mask the final volume in some smart way...
# To filter the volume
if self.useMaxRes:
self.runJob('xmipp_transform_filter',
'-i %s --fourier low_pass %f --sampling %f' % \
(volFn, self.maxResolution.get(), self.TsCurrent),
numberOfMpi=1)
if not self.keepIntermediate:
cleanPath(prevVolFn, iterDir)
def reconstructionStep(self, numberOfImages, fractionCounter, iteration,
debugging, fnNewImgMd, particlesMd):
fnRoot = self._getExtraPath("fraction%02d" % fractionCounter)
Ts = self.inputParticles.get().getSamplingRate()
# for noise
fnRootN = self._getExtraPath("Nfraction%02d" % fractionCounter)
for i in range(0, 2):
fnImgs = fnRoot + "_images_%02d_%02d.xmd" % (i, iteration)
if self.doResize.get():
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" %
(fnNewImgMd, fnImgs, numberOfImages),
numberOfMpi=1)
else:
self.runJob("xmipp_metadata_utilities",
"-i %s -o %s --operate random_subset %d" %
(particlesMd, fnImgs, numberOfImages),
numberOfMpi=1)
params = ' -i %s' % fnImgs
params += ' -o %s' % fnRoot + "_%02d_%02d.vol" % (i, iteration)
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --max_resolution %0.3f' % self.maxRes
params += ' --padding 2'
params += ' --fast'
params += ' --sampling %f' % Ts
if self.useGpu.get():
#AJ to make it work with and without queue system
if self.numberOfMpi.get() > 1:
N_GPUs = len((self.gpuList.get()).split(','))
params += ' -gpusPerNode %d' % N_GPUs
params += ' -threadsPerGPU %d' % max(
self.numberOfThreads.get(), 4)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
if self.numberOfMpi.get() == 1:
params += " --device %s" % (GpuListCuda)
params += ' --thr %d' % self.numberOfThreads.get()
if self.numberOfMpi.get() > 1:
self.runJob('xmipp_cuda_reconstruct_fourier',
params,
numberOfMpi=len(
(self.gpuList.get()).split(',')) + 1)
else:
self.runJob('xmipp_cuda_reconstruct_fourier', params)
else:
self.runJob('xmipp_reconstruct_fourier_accel', params)
# for noise
if self.doNoise.get():
noiseStk = fnRoot + "_noises_%02d.stk" % i
self.runJob("xmipp_image_convert",
"-i %s -o %s" % (fnImgs, noiseStk),
numberOfMpi=1)
self.runJob("xmipp_image_operate", "-i %s --mult 0" % noiseStk)
self.runJob("xmipp_transform_add_noise",
"-i %s --type gaussian 3" % noiseStk,
numberOfMpi=1)
fnImgsNL = fnRoot + "_noisesL_%02d.xmd" % i
noiseStk2 = fnRoot + "_noises2_%02d.stk" % i
self.runJob("xmipp_image_convert",
"-i %s -o %s --save_metadata_stack %s" %
(noiseStk, noiseStk2, fnImgsNL),
numberOfMpi=1)
fnImgsNoiseOld = fnRoot + "_noisesOld_%02d.xmd" % i
fnImgsN = fnRoot + "_noises_%02d_%02d.xmd" % (i, iteration)
self.runJob("xmipp_metadata_utilities",
'-i %s -o %s --operate drop_column "image"' %
(fnImgs, fnImgsNoiseOld),
numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set merge %s -o %s" %
(fnImgsNL, fnImgsNoiseOld, fnImgsN),
numberOfMpi=1)
# alignment gaussian noise
fnImgsAlign = self._getExtraPath("Nfraction_alignment%02d" %
fractionCounter)
fnImgsAlignN = fnImgsAlign + "_%02d_%02d.xmd" % (i, iteration)
args = "-i %s -o %s -r %s --Ri 0 --Ro -1 --mem 2 --append " % (
fnImgsN, fnImgsAlignN,
self._getExtraPath("Ref_Projections.stk"))
self.runJob('xmipp_angular_projection_matching',
args,
numberOfMpi=self.numberOfMpi.get())
# numberOfMpi = self.numberOfMpi.get() * self.numberOfThreads.get())
params = ' -i %s' % fnImgsAlignN
params += ' -o %s' % fnRootN + "_%02d_%02d.vol" % (i,
iteration)
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --max_resolution %0.3f' % self.maxRes
params += ' --padding 2'
params += ' --sampling %f' % Ts
if self.useGpu.get():
#AJ to make it work with and without queue system
if self.numberOfMpi.get() > 1:
N_GPUs = len((self.gpuList.get()).split(','))
params += ' -gpusPerNode %d' % N_GPUs
params += ' -threadsPerGPU %d' % max(
self.numberOfThreads.get(), 4)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
if self.numberOfMpi.get() == 1:
params += " --device %s" % (GpuListCuda)
params += ' --thr %d' % self.numberOfThreads.get()
if self.numberOfMpi.get() > 1:
self.runJob('xmipp_cuda_reconstruct_fourier',
params,
numberOfMpi=len(
(self.gpuList.get()).split(',')) + 1)
else:
self.runJob('xmipp_cuda_reconstruct_fourier', params)
else:
self.runJob('xmipp_reconstruct_fourier_accel', params)
self.runJob('xmipp_resolution_fsc',
"--ref %s -i %s -o %s --sampling_rate %f" % \
(fnRoot + "_00_%02d.vol" % iteration,
fnRoot + "_01_%02d.vol" % iteration,
fnRoot + "_fsc_%02d.xmd" % iteration, Ts),
numberOfMpi=1)
mdFSC = emlib.MetaData(fnRoot + "_fsc_%02d.xmd" % iteration)
for id in mdFSC:
fscValue = mdFSC.getValue(emlib.MDL_RESOLUTION_FRC, id)
maxFreq = mdFSC.getValue(emlib.MDL_RESOLUTION_FREQREAL, id)
if fscValue < 0.5:
break
fh = open(fnRoot + "_freq.txt", "a")
fh.write("%f\n" % maxFreq)
fh.close()
# for noise
if self.doNoise.get():
self.runJob('xmipp_resolution_fsc',
"--ref %s -i %s -o %s --sampling_rate %f" %
(fnRootN + "_00_%02d.vol" % iteration,
fnRootN + "_01_%02d.vol" % iteration,
fnRootN + "_fsc_%02d.xmd" % iteration, Ts),
numberOfMpi=1)
cleanPattern(fnRoot + "_noises_0?_0?.xmd")
cleanPattern(fnRoot + "_noisesOld_0?.xmd")
cleanPattern(fnRoot + "_noisesL_0?.xmd")
cleanPattern(fnRoot + "_noises2_0?.stk")
mdFSCN = emlib.MetaData(fnRootN + "_fsc_%02d.xmd" % iteration)
for id in mdFSCN:
fscValueN = mdFSCN.getValue(emlib.MDL_RESOLUTION_FRC, id)
maxFreqN = mdFSCN.getValue(emlib.MDL_RESOLUTION_FREQREAL, id)
if fscValueN < 0.5:
break
fhN = open(fnRootN + "_freq.txt", "a")
fhN.write("%f\n" % maxFreqN)
fhN.close()
if not debugging:
cleanPattern(fnRoot + "_0?_0?.vol")
cleanPattern(fnRoot + "_images_0?_0?.xmd")
cleanPattern(fnRoot + "_fsc_0?.xmd")
cleanPattern(fnRoot + "_noises_0?.stk")
if self.doNoise.get():
cleanPattern(fnImgsAlign + "_0?_0?.xmd")
cleanPattern(fnRootN + "_0?_0?.vol")
cleanPattern(fnRootN + "_fsc_0?.xmd")
def significantStep(self, iterNumber, alpha):
iterDir = self._getTmpPath('iter%03d' % iterNumber)
makePath(iterDir)
prevVolFn = self.getIterVolume(iterNumber - 1)
volFn = self.getIterVolume(iterNumber)
anglesFn = self._getExtraPath('angles_iter%03d.xmd' % iterNumber)
t = Timer()
t.tic()
if self.useGpu.get() and iterNumber > 1:
# Generate projections
fnGalleryRoot = join(iterDir, "gallery")
args = "-i %s -o %s.stk --sampling_rate %f --sym %s " \
"--compute_neighbors --angular_distance -1 " \
"--experimental_images %s --min_tilt_angle %f " \
"--max_tilt_angle %f -v 0 --perturb %f " % \
(prevVolFn, fnGalleryRoot, self.angularSampling.get(),
self.symmetryGroup, self.imgsFn, self.minTilt, self.maxTilt,
math.sin(self.angularSampling.get()) / 4)
self.runJob("xmipp_angular_project_library ", args, numberOfMpi=1)
if self.trueSymsNo != 0:
alphaApply = (alpha * self.trueSymsNo) / 2
else:
alphaApply = alpha / 2
from pwem.emlib.metadata import getSize
N = int(getSize(fnGalleryRoot + '.doc') * alphaApply * 2)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
args = '-i %s -r %s.doc -o %s --keepBestN %f --dev %s ' % \
(self.imgsFn, fnGalleryRoot, anglesFn, N, GpuListCuda)
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
cleanPattern(fnGalleryRoot + "*")
else:
args = self.getSignificantArgs(self.imgsFn)
args += ' --odir %s' % iterDir
args += ' --alpha0 %f --alphaF %f' % (alpha, alpha)
args += ' --dontCheckMirrors '
if iterNumber == 1:
if self.thereisRefVolume:
args += " --initvolumes " + \
self._getExtraPath('input_volumes.xmd')
else:
args += " --numberOfVolumes 1"
else:
args += " --initvolumes %s" % prevVolFn
self.runJob("xmipp_reconstruct_significant", args)
moveFile(os.path.join(iterDir, 'angles_iter001_00.xmd'), anglesFn)
t.toc('Significant took: ')
reconsArgs = ' -i %s --fast' % anglesFn
reconsArgs += ' -o %s' % volFn
reconsArgs += ' --weight -v 0 --sym %s ' % self.symmetryGroup
print("Number of images for reconstruction: ",
metadata.getSize(anglesFn))
t.tic()
if self.useGpu.get():
cudaReconsArgs = reconsArgs
#AJ to make it work with and without queue system
if self.numberOfMpi.get() > 1:
N_GPUs = len((self.gpuList.get()).split(','))
cudaReconsArgs += ' -gpusPerNode %d' % N_GPUs
cudaReconsArgs += ' -threadsPerGPU %d' % max(
self.numberOfThreads.get(), 4)
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
cudaReconsArgs += ' --thr %s' % self.numberOfThreads.get()
if self.numberOfMpi.get() == 1:
cudaReconsArgs += ' --device %s' % (GpuListCuda)
if self.numberOfMpi.get() > 1:
self.runJob('xmipp_cuda_reconstruct_fourier',
cudaReconsArgs,
numberOfMpi=len(
(self.gpuList.get()).split(',')) + 1)
else:
self.runJob('xmipp_cuda_reconstruct_fourier', cudaReconsArgs)
else:
self.runJob("xmipp_reconstruct_fourier_accel", reconsArgs)
t.toc('Reconstruct fourier took: ')
# Center the volume
fnSym = self._getExtraPath('volumeSym_%03d.vol' % iterNumber)
self.runJob("xmipp_transform_mirror",
"-i %s -o %s --flipX" % (volFn, fnSym),
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipY" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_transform_mirror",
"-i %s --flipZ" % fnSym,
numberOfMpi=1)
self.runJob("xmipp_image_operate",
"-i %s --plus %s" % (fnSym, volFn),
numberOfMpi=1)
self.runJob("xmipp_volume_align",
'--i1 %s --i2 %s --local --apply' % (fnSym, volFn),
numberOfMpi=1)
cleanPath(fnSym)
# To mask the volume
xdim = self.inputSet.get().getDimensions()[0]
maskArgs = "-i %s --mask circular %d -v 0" % (volFn, -xdim / 2)
self.runJob('xmipp_transform_mask', maskArgs, numberOfMpi=1)
# TODO mask the final volume in some smart way...
# To filter the volume
if self.useMaxRes:
self.runJob('xmipp_transform_filter',
'-i %s --fourier low_pass %f --sampling %f' % \
(volFn, self.maxResolution.get(), self.TsCurrent),
numberOfMpi=1)
if not self.keepIntermediate:
cleanPath(prevVolFn, iterDir)
if self.thereisRefVolume:
cleanPath(self._getExtraPath('filteredVolume.vol'))
