def run_extract(suffix=''):
args = ('--extract --o subparticles --extract_size %s --coord_files '
'"%s/particles%s_??????.star"') % (subpart_size, output, suffix)
run_command(cmd + args, "/dev/null")
print(" Cleaning up temporary files...")
run_command("rm subparticles.star")
if deleteParticles:
pwutils.cleanPattern('%s/particles%s_??????.mrc' % (output, suffix))
def _showProjMatchLibrary(self, paramName=None):
#map stack position with ref number
list = []
mdIn = emlib.MetaData()
mdOut = emlib.MetaData()
cleanPattern(self.protocol._getExtraPath('references_library*'))
for ref3d in self._refsList:
for it in self._iterations:
convert_refno_to_stack_position = {}
file_name = self.protocol._getFileName(
'projectLibrarySampling', iter=it, ref=ref3d)
file_nameReferences = 'projectionDirections@' + file_name
#last reference name
if exists(file_name):
mdReferences = emlib.MetaData(file_nameReferences)
mdReferencesSize = mdReferences.size()
for id in mdReferences:
convert_refno_to_stack_position[mdReferences.getValue(
emlib.MDL_NEIGHBOR, id)] = id
file_nameAverages = self.protocol._getFileName(
'outClassesXmd', iter=it, ref=ref3d)
if exists(file_nameAverages):
# print("OutClassesXmd %s" % OutClassesXmd)
mdIn.read(file_nameAverages)
mdOut.clear()
for i in mdIn:
# id1=mdOut.addObject()
# mdOut.setValue(MDL_IMAGE,mdIn.getValue(MDL_IMAGE,i),id1)
ref2D = mdIn.getValue(emlib.MDL_REF, i)
file_references = self.protocol._getFileName(
'projectLibraryStk', iter=it, ref=ref3d)
file_reference = emlib.FileName()
file_reference.compose(
convert_refno_to_stack_position[ref2D],
file_references)
id2 = mdOut.addObject()
mdOut.setValue(emlib.MDL_IMAGE, file_reference,
id2)
if mdOut.size() == 0:
print("Empty metadata: ", file_nameReferences)
else:
file_nameReferences = self.protocol._getExtraPath(
'references_library.xmd')
sfn = createUniqueFileName(file_nameReferences)
file_nameReferences = 'projectionDirections@' + sfn
mdOut.write(file_nameReferences)
list.append(
self.createDataView(file_nameReferences))
else:
print("File %s does not exist" % file_name)
return list
def extractParticlesStep(self, micId, baseMicName, fnCTF,
micrographToExtract, micOps, doInvert,
normalizeArgs, doBorders):
""" Extract particles from one micrograph """
outputRoot = str(self._getExtraPath(baseMicName))
fnPosFile = self._getExtraPath(baseMicName + ".pos")
# If it has coordinates extract the particles
particlesMd = 'particles@%s' % fnPosFile
boxSize = self.boxSize.get()
boxScale = self.getBoxScale()
print "boxScale: ", boxScale
if exists(fnPosFile):
# Apply first all operations required for the micrograph
for program, args in micOps:
self.runJob(program, args)
args = " -i %s --pos %s" % (micrographToExtract, particlesMd)
args += " -o %s --Xdim %d" % (outputRoot, boxSize)
if doInvert:
args += " --invert"
if fnCTF:
args += " --ctfparam " + fnCTF
if doBorders:
args += " --fillBorders"
self.runJob("xmipp_micrograph_scissor", args)
# Normalize
if normalizeArgs:
self.runJob('xmipp_transform_normalize',
'-i %s.stk %s' % (outputRoot, normalizeArgs))
else:
self.warning("The micrograph %s hasn't coordinate file! " %
baseMicName)
self.warning("Maybe you picked over a subset of micrographs")
# Let's clean the temporary mrc micrographs
if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
def _showProjMatchLibrary(self, paramName=None):
#map stack position with ref number
list = []
mdIn = xmipp.MetaData()
mdOut = xmipp.MetaData()
cleanPattern(self.protocol._getTmpPath('references_library*'))
for ref3d in self._refsList:
for it in self._iterations:
convert_refno_to_stack_position = {}
file_name = self.protocol._getFileName('projectLibrarySampling', iter=it, ref=ref3d)
file_nameReferences = 'projectionDirections@' + file_name
#last reference name
if exists(file_name):
mdReferences = xmipp.MetaData(file_nameReferences)
mdReferencesSize = mdReferences.size()
for id in mdReferences:
convert_refno_to_stack_position[mdReferences.getValue(xmipp.MDL_NEIGHBOR,id)]=id
file_nameAverages = self.protocol._getFileName('outClassesXmd', iter=it, ref=ref3d)
if exists(file_nameAverages):
#print "OutClassesXmd", OutClassesXmd
mdIn.read(file_nameAverages)
mdOut.clear()
for i in mdIn:
#id1=mdOut.addObject()
#mdOut.setValue(MDL_IMAGE,mdIn.getValue(MDL_IMAGE,i),id1)
ref2D = mdIn.getValue(xmipp.MDL_REF,i)
file_references = self.protocol._getFileName('projectLibraryStk', iter=it, ref=ref3d)
file_reference = xmipp.FileName()
file_reference.compose(convert_refno_to_stack_position[ref2D],file_references)
id2=mdOut.addObject()
mdOut.setValue(xmipp.MDL_IMAGE, file_reference, id2)
if mdOut.size() == 0:
print "Empty metadata: ", file_nameReferences
else:
file_nameReferences = self.protocol._getTmpPath('references_library.xmd')
sfn = createUniqueFileName(file_nameReferences)
file_nameReferences = 'projectionDirections@' + sfn
mdOut.write(file_nameReferences)
list.append(self.createDataView(file_nameReferences))
else:
print "File %s does not exist" % file_name
return list
def extractParticlesStep(self, micId, baseMicName, fnCTF,
micrographToExtract, micOps,
doInvert, normalizeArgs, doBorders):
""" Extract particles from one micrograph """
outputRoot = str(self._getExtraPath(baseMicName))
fnPosFile = self._getExtraPath(baseMicName + ".pos")
# If it has coordinates extract the particles
particlesMd = 'particles@%s' % fnPosFile
boxSize = self.boxSize.get()
boxScale = self.getBoxScale()
print "boxScale: ", boxScale
if exists(fnPosFile):
# Apply first all operations required for the micrograph
for program, args in micOps:
self.runJob(program, args)
args = " -i %s --pos %s" % (micrographToExtract, particlesMd)
args += " -o %s --Xdim %d" % (outputRoot, boxSize)
if doInvert:
args += " --invert"
if fnCTF:
args += " --ctfparam " + fnCTF
if doBorders:
args += " --fillBorders"
self.runJob("xmipp_micrograph_scissor", args)
# Normalize
if normalizeArgs:
self.runJob('xmipp_transform_normalize',
'-i %s.stk %s' % (outputRoot, normalizeArgs))
else:
self.warning("The micrograph %s hasn't coordinate file! "
% baseMicName)
self.warning("Maybe you picked over a subset of micrographs")
# Let's clean the temporary mrc micrographs
if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
def _restimateCTF(self, ctfId):
""" Run ctftilt with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
pwutils.cleanPath(psdFile)
try:
program, args = self._getRecalCommand(ctfModel,
micFn=micFnMrc,
ctftiltOut=out,
ctftiltPSD=psdFile)
self.runJob(program, args)
except Exception as ex:
print >> sys.stderr, "ctftilt has failed with micrograph %s" % micFnMrc
pwutils.cleanPattern(micFnMrc)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = out
self._params['ctffindPSD'] = psdFile
pwutils.cleanPath(psdFile)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
pwutils.cleanPattern(micFnMrc)
def _createNewCtfModel(self, mic):
micDir = self._getMicrographDir(mic)
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
ctfModel2 = em.CTFModel()
readCtfModel(ctfModel2, out, ctf4=self.useCtffind4.get())
ctfModel2.setPsdFile(psdFile)
ctfModel2.setMicrograph(mic)
return ctfModel2
def _createOutputStep(self):
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(self.inputMics)
defocusList = []
def _extractMicrograph(self, mic, doInvert, normalizeArgs, doBorders):
""" Extract particles from one micrograph """
fnLast = mic.getFileName()
baseMicName = pwutils.removeBaseExt(fnLast)
outputRoot = str(self._getExtraPath(baseMicName))
fnPosFile = self._getMicPos(mic)
boxSize = self.boxSize.get()
downFactor = self.downFactor.get()
patchSize = self.patchSize.get() if self.patchSize.get() > 0 \
else int(boxSize*1.5*downFactor)
particlesMd = 'particles@%s' % fnPosFile
# If it has coordinates extract the particles
if exists(fnPosFile):
# Create a list with micrographs operations (programs in xmipp) and
# the required command line parameters (except input/ouput files)
micOps = []
# Compute the variance and Gini coeff. of the part. and mic., resp.
args = '--pos %s' % fnPosFile
args += ' --mic %s' % fnLast
args += ' --patchSize %d' % patchSize
self.runJob('xmipp_coordinates_noisy_zones_filter', args)
def getMicTmp(suffix):
return self._getTmpPath(baseMicName + suffix)
# Check if it is required to downsample our micrographs
if self.notOne(downFactor):
fnDownsampled = getMicTmp("_downsampled.xmp")
args = "-i %s -o %s --step %f --method fourier"
self.runJob('xmipp_transform_downsample',
args % (fnLast, fnDownsampled, downFactor))
fnLast = fnDownsampled
if self.doRemoveDust:
fnNoDust = getMicTmp("_noDust.xmp")
args = " -i %s -o %s --bad_pixels outliers %f"
self.runJob('xmipp_transform_filter',
args % (fnLast, fnNoDust, self.thresholdDust))
fnLast = fnNoDust
if self._useCTF():
# We need to write a Xmipp ctfparam file
# to perform the phase flip on the micrograph
fnCTF = self._getTmpPath("%s.ctfParam" % baseMicName)
micrographToCTFParam(mic, fnCTF)
# Insert step to flip micrograph
if self.doFlip:
fnFlipped = getMicTmp('_flipped.xmp')
args = " -i %s -o %s --ctf %s --sampling %f"
self.runJob(
'xmipp_ctf_phase_flip', args %
(fnLast, fnFlipped, fnCTF, self._getNewSampling()))
fnLast = fnFlipped
else:
fnCTF = None
args = " -i %s --pos %s" % (fnLast, particlesMd)
args += " -o %s --Xdim %d" % (outputRoot, boxSize)
if doInvert:
args += " --invert"
if fnCTF:
args += " --ctfparam " + fnCTF
if doBorders:
args += " --fillBorders"
self.runJob("xmipp_micrograph_scissor", args)
# Normalize
if normalizeArgs:
self.runJob('xmipp_transform_normalize',
'-i %s.stk %s' % (outputRoot, normalizeArgs))
else:
self.warning("The micrograph %s hasn't coordinate file! " %
baseMicName)
self.warning("Maybe you picked over a subset of micrographs")
# Let's clean the temporary mrc micrographs
if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
def refineAnglesStep(self):
fnTmpDir = self._getTmpPath()
fnDirectional = self._getDirectionalClassesFn()
inputParticles = self.inputParticles.get()
newTs = readInfoField(self._getExtraPath(), "sampling",
emlib.MDL_SAMPLINGRATE)
newXdim = readInfoField(self._getExtraPath(), "size", emlib.MDL_XSIZE)
# Generate projections
fnGallery = join(fnTmpDir, "gallery.stk")
fnGalleryMd = join(fnTmpDir, "gallery.doc")
fnVol = self._getInputVolFn()
args = "-i %s -o %s --sampling_rate %f --sym %s" % \
(fnVol, fnGallery, 5.0, self.symmetryGroup)
args += " --compute_neighbors --angular_distance -1 --experimental_images %s" % fnDirectional
self.runJob("xmipp_angular_project_library",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
# Global angular assignment
maxShift = 0.15 * newXdim
fnAngles = join(fnTmpDir, "angles_iter001_00.xmd")
if not self.useGpu.get():
args = '-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0' % \
(fnDirectional, fnGalleryMd, maxShift, fnTmpDir)
self.runJob('xmipp_reconstruct_significant',
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
else:
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
args = '-i %s -r %s -o %s --dev %s ' % (fnDirectional, fnGalleryMd,
fnAngles, GpuListCuda)
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --operate drop_column ref" % fnAngles,
numberOfMpi=1)
self.runJob("xmipp_metadata_utilities",
"-i %s --set join %s ref2" % (fnAngles, fnDirectional),
numberOfMpi=1)
# Local angular assignment
fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
args = "-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 " % \
(fnAngles, fnAnglesLocalStk, newTs, newXdim / 2, 2, fnVol,
self.targetResolution)
args += " --optimizeShift --max_shift %f" % maxShift
args += " --optimizeAngles --max_angular_change %f" % self.angularDistance
self.runJob("xmipp_angular_continuous_assign2",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
moveFile(self._getPath("directional_local_classes.xmd"),
self._getDirectionalClassesFn())
cleanPattern(self._getExtraPath("direction_*"))