def convertToMrc(fn):
""" Convert from a format that is not read by Relion
to mrc format.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'mrc'))
ih.convert(fn, newFn)
return newFn
def readSetOfCoordinates(outputDir, micSet, coordSet):
""" Read from Bsoft .star files.
Params:
outputDir: the directory where the .star files are.
micSet: the SetOfMicrographs to associate the .star, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
addBsoftLabelAliases()
boxSize = 0
for mic in micSet:
outputFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'star'))
#scipionPosFile = join(outputDir, "scipion_" + replaceBaseExt(mic.getFileName(), 'pos'))
if exists(outputFile):
posMd = xmipp.MetaData(outputFile)
for objId in posMd:
coord = rowToCoordinate(rowFromMd(posMd, objId))
boxSize = 2 * posMd.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE, objId)
coord.setMicrograph(mic)
coord.setX(coord.getX())
coord.setY(coord.getY())
coordSet.append(coord)
# Add an unique ID that will be propagated to particles
posMd.setValue(xmipp.MDL_ITEM_ID, long(coord.getObjId()), objId)
# if not posMd.isEmpty():
# posMd.write("particles@%s" % scipionPosFile)
#reading origin.x value and converting to particle size, can change, we take last value
coordSet.setBoxSize(boxSize)
def readSetOfCoordinates(outputDir, partSet, coordSet):
"""This method is used to store an edition of the coordinates obtained by
localized reconstruction.
Read from Xmipp .pos files.
Params:
outputDir: the directory where the .pos files are.
It is also expected a file named: config.xmd
in this directory where the box size can be read.
micSet: the SetOfMicrographs to associate the .pos, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the config.xmd metadata
configfile = join(outputDir, 'config.xmd')
if exists(configfile):
mdFn = md.MetaData('properties@' + join(outputDir, 'config.xmd'))
boxSize = mdFn.getValue(md.MDL_PICKING_PARTICLE_SIZE,
mdFn.firstObject())
coordSet.setBoxSize(boxSize)
for part in partSet:
baseFn = "%06d_at_%s" % (part.getIndex(),
replaceBaseExt(part.getFileName(), 'pos'))
posFile = join(outputDir, baseFn)
readCoordinates(part, posFile, coordSet, outputDir)
coordSet._xmippMd = String(outputDir)
def linkMrcToMrcs(fn):
""" Just create a link named .mrc to Eman understand
that it is a mrc binary stack.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'mrcs'))
createLink(fn, newFn)
return newFn
def getImgIdFromCoord(self, coordId):
""" Get the image id from the related coordinate id. """
'%s:%06d'
parts = coordId.split(':')
imgFn = self._getExtraPath(replaceBaseExt(parts[0], "stk"))
return '%06d@%s' % (int(parts[1]), imgFn)
def convertStack(fn):
""" Convert from a format that is not read by Relion
to an spider stack.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'stk'))
ih.convertStack(fn, newFn)
return newFn
def getImgIdFromCoord(self, coordId):
""" Get the image id from the related coordinate id. """
'%s:%06d'
parts = coordId.split(':')
imgFn = self._getExtraPath(replaceBaseExt(parts[0], "stk"))
return '%06d@%s' %(int(parts[1]), imgFn)
def readSetOfCoordinates(outputDir, micSet, coordSet):
""" Read from Bsoft .star files.
Params:
outputDir: the directory where the .star files are.
micSet: the SetOfMicrographs to associate the .star, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
addBsoftLabelAliases()
boxSize = 0
for mic in micSet:
outputFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'star'))
#scipionPosFile = join(outputDir, "scipion_" + replaceBaseExt(mic.getFileName(), 'pos'))
if exists(outputFile):
posMd = xmipp.MetaData(outputFile)
for objId in posMd:
coord = rowToCoordinate(rowFromMd(posMd, objId))
boxSize = 2 * posMd.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE,
objId)
coord.setMicrograph(mic)
coord.setX(coord.getX())
coord.setY(coord.getY())
coordSet.append(coord)
# Add an unique ID that will be propagated to particles
posMd.setValue(xmipp.MDL_ITEM_ID, long(coord.getObjId()),
objId)
# if not posMd.isEmpty():
# posMd.write("particles@%s" % scipionPosFile)
#reading origin.x value and converting to particle size, can change, we take last value
coordSet.setBoxSize(boxSize)
def createBinaryLink(fn):
""" Just create a link named .mrcs to Relion understand
that it is a binary stack file and not a volume.
"""
newFn = join(outputDir, replaceBaseExt(fn, extension))
createLink(fn, newFn)
return newFn
def convertStack(fn):
""" Convert from a format that is not read by Relion
to an spider stack.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'stk'))
ih.convertStack(fn, newFn)
return newFn
def linkMrcToMrcs(fn):
""" Just create a link named .mrc to Eman understand
that it is a mrc binary stack.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'mrcs'))
createLink(fn, newFn)
return newFn
def convertToMrc(fn):
""" Convert from a format that is not read by Relion
to mrc format.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'mrc'))
ih.convert(fn, newFn)
return newFn
def readSetOfCoordinates(outputDir, partSet, coordSet):
"""This method is used to store an edition of the coordinates obtained by
localized reconstruction.
Read from Xmipp .pos files.
Params:
outputDir: the directory where the .pos files are.
It is also expected a file named: config.xmd
in this directory where the box size can be read.
micSet: the SetOfMicrographs to associate the .pos, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the config.xmd metadata
configfile = join(outputDir, 'config.xmd')
if exists(configfile):
mdFn = md.MetaData('properties@' + join(outputDir, 'config.xmd'))
boxSize = mdFn.getValue(md.MDL_PICKING_PARTICLE_SIZE,
mdFn.firstObject())
coordSet.setBoxSize(boxSize)
for part in partSet:
baseFn = "%06d_at_%s" % (part.getIndex(),
replaceBaseExt(part.getFileName(), 'pos'))
posFile = join(outputDir, baseFn)
readCoordinates(part, posFile, coordSet, outputDir)
coordSet._xmippMd = String(outputDir)
def _estimateCTF(self, micFn, micDir, micName):
""" Run the estimate CTF program """
localParams = self.__params.copy()
if self.doInitialCTF:
if self.ctfDict[micName]>0:
localParams['defocusU'] = self.ctfDict[micName]
localParams['defocus_range'] = 0.01*self.ctfDict[micName]
else:
localParams['defocusU']=(self._params['maxDefocus']+self._params['minDefocus'])/2
localParams['defocus_range']=(self._params['maxDefocus']-self._params['minDefocus'])/2
# Create micrograph dir under extra directory
makePath(micDir)
if not exists(micDir):
raise Exception("No created dir: %s " % micDir)
finalName = micFn
ctfDownFactor = self.ctfDownFactor.get()
downsampleList = [ctfDownFactor]
if self.doCTFAutoDownsampling:
downsampleList.append(ctfDownFactor+1)
if ctfDownFactor >= 2:
downsampleList.append(ctfDownFactor-1)
else:
if ctfDownFactor > 1:
downsampleList.append((ctfDownFactor+1)/2)
deleteTmp=""
for downFactor in downsampleList:
# Downsample if necessary
if downFactor != 1:
#Replace extension by 'mrc' cause there are some formats that cannot be written (such as dm3)
finalName = self._getTmpPath(replaceBaseExt(micFn, 'mrc'))
self.runJob("xmipp_transform_downsample","-i %s -o %s --step %f --method fourier" % (micFn, finalName, downFactor))
deleteTmp=finalName
# Update _params dictionary with mic and micDir
localParams['micFn'] = finalName
localParams['micDir'] = self._getFileName('prefix', micDir=micDir)
localParams['samplingRate'] = self.inputMics.getSamplingRate() * downFactor
# CTF estimation with Xmipp
try:
self.runJob(self._program, self._args % localParams+" --downSamplingPerformed %f"%downFactor)
# mdCTF = md.RowMetaData(self._getFileName('ctfparam', micDir=micDir))
except Exception:
break
# Check the quality of the estimation and reject it necessary
if self.evaluateSingleMicrograph(micFn,micDir):
break
if deleteTmp != "":
cleanPath(deleteTmp)
def executeDogpickerStep(self, inputMic, args):
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
# Run the command with formatted parameters
self._log.info('Launching: ' + program + ' ' + args)
self.runJob(program, args)
def executeDogpickerStep(self, inputMic, args):
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
# Run the command with formatted parameters
self._log.info('Launching: ' + program + ' ' + args)
self.runJob(program, args)
def writeSetOfCoordinates(posDir, coordSet):
""" Write a pos file on metadata format for each micrograph
on the coordSet.
Params:
posDir: the directory where the .pos files will be written.
coordSet: the SetOfCoordinates that will be read.
"""
posFiles = []
boxSize = coordSet.getBoxSize() or 100
# Write pos metadatas (one per micrograph)
for mic in coordSet.iterMicrographs():
micName = mic.getFileName()
posFn = join(posDir, replaceBaseExt(micName, "pos"))
md = xmipp.MetaData()
for coord in coordSet.iterCoordinates(micrograph=mic):
objId = md.addObject()
coordRow = XmippMdRow()
coordinateToRow(coord, coordRow)
coordRow.writeToMd(md, objId)
if not md.isEmpty():
md2 = xmipp.MetaData()
objId = md2.addObject()
md2.setValue(xmipp.MDL_PICKING_MICROGRAPH_STATE, 'Manual', objId)
# Write header block
md2.write('header@%s' % posFn)
# Write particles block
md.write('particles@%s' % posFn, xmipp.MD_APPEND)
posFiles.append(posFn)
# Write config.xmd metadata
configFn = join(posDir, 'config.xmd')
md = xmipp.MetaData()
# Write properties block
objId = md.addObject()
micName = removeBaseExt(micName)
md.setValue(xmipp.MDL_MICROGRAPH, str(micName), objId)
#md.setValue(xmipp.MDL_COLOR, int(-16776961), objId)
md.setValue(xmipp.MDL_PICKING_PARTICLE_SIZE, int(boxSize), objId)
md.setValue(xmipp.MDL_PICKING_STATE, 'Manual', objId)
md.write('properties@%s' % configFn)
# Write filters block
md = xmipp.MetaData()
objId = md.addObject()
md.setValue(xmipp.MDL_MACRO_CMD, 'Gaussian_Blur...', objId)
md.setValue(xmipp.MDL_MACRO_CMD_ARGS, 'sigma=2', objId)
md.write('filters@%s' % configFn, xmipp.MD_APPEND)
return posFiles
def readSetOfCoordinates(workDir, micSet, coordSet):
""" Read from Appion .txt files.
It is expected a file named: base.txt under the workDir.
Params:
workDir: where the Appion dogpicker output files are located.
micSet: the SetOfMicrographs to associate the .txt, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
for mic in micSet:
micCoordFn = join(workDir, replaceBaseExt(mic.getFileName(), 'txt'))
readCoordinates(mic, micCoordFn, coordSet)
def readSetOfCoordinates(workDir, micSet, coordSet):
""" Read from Appion .txt files.
It is expected a file named: base.txt under the workDir.
Params:
workDir: where the Appion dogpicker output files are located.
micSet: the SetOfMicrographs to associate the .txt, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
for mic in micSet:
micCoordFn = join(workDir, replaceBaseExt(mic.getFileName(), 'txt'))
readCoordinates(mic, micCoordFn, coordSet)
def readCoordinates(mic, fileName, coordsSet, outputDir):
posMd = readPosCoordinates(fileName)
posMd.addLabel(xmipp.MDL_ITEM_ID)#TODO: CHECK IF THIS LABEL IS STILL NECESSARY
for objId in posMd:
coord = rowToCoordinate(rowFromMd(posMd, objId))
coord.setMicrograph(mic)
coord.setX(coord.getX())
coord.setY(coord.getY())
coordsSet.append(coord)
# Add an unique ID that will be propagated to particles
posMd.setValue(xmipp.MDL_ITEM_ID, long(coord.getObjId()), objId)
if not posMd.isEmpty():
scipionPosFile = join(outputDir, "scipion_" + replaceBaseExt(mic.getFileName(), 'pos'))
posMd.write("particles@%s" % scipionPosFile)
def createMetadataImageStep(self):
#Create images.xmd metadata
fnImages = self._getOutputImgMd()
imgsXmd = md.MetaData()
posFiles = glob(self._getExtraPath('*.pos'))
for posFn in posFiles:
xmdFn = self._getExtraPath(replaceBaseExt(posFn, "xmd"))
if exists(xmdFn):
mdFn = md.MetaData(xmdFn)
mdPos = md.MetaData('particles@%s' % posFn)
mdPos.merge(mdFn)
#imgSet.appendFromMd(mdPos)
imgsXmd.unionAll(mdPos)
else:
self.warning("The coord file %s wasn't used to extract! Maybe you are extracting over a subset of micrographs" % basename(posFn))
imgsXmd.write(fnImages)
def getUniqueFileName(fn, extension):
""" Get an unique file for either link or convert files.
It is possible that the base name overlap if they come
from different runs. (like partices.mrcs after relion preprocess)
"""
newFn = join(outputDir, replaceBaseExt(fn, extension))
newRoot = removeExt(newFn)
values = filesDict.values()
counter = 1
while newFn in values:
counter += 1
newFn = '%s_%05d.%s' % (newRoot, counter, extension)
return newFn
def getUniqueFileName(fn, extension):
""" Get an unique file for either link or convert files.
It is possible that the base name overlap if they come
from different runs. (like partices.mrcs after relion preprocess)
"""
newFn = join(outputDir, replaceBaseExt(fn, extension))
newRoot = removeExt(newFn)
values = filesDict.values()
counter = 1
while newFn in values:
counter += 1
newFn = '%s_%05d.%s' % (newRoot, counter, extension)
return newFn
def runTemplate(inputScript, paramsDict, log=None, cwd=None):
""" This function will create a valid Imagic script
by copying the template and replacing the values in dictionary.
After the new file is read, the Imagic interpreter is invoked.
Usually the execution should be done where the results will
be left.
"""
outputScript = replaceBaseExt(inputScript, 'b')
if cwd is not None:
outputScript = join(cwd, outputScript)
# First write the script from the template with the substitutions
writeScript(inputScript, outputScript, paramsDict)
# Then proceed to run the script
runScript(outputScript, log, cwd)
def runTemplate(inputScript, ext, paramsDict, log=None, cwd=None):
""" This function will create a valid Spider script
by copying the template and replacing the values in dictionary.
After the new file is read, the Spider interpreter is invoked.
Usually the execution should be done where the results will
be left.
"""
outputScript = replaceBaseExt(inputScript, ext)
if cwd is not None:
outputScript = join(cwd, outputScript)
# First write the script from the template with the substitutions
writeScript(inputScript, outputScript, paramsDict)
# Then proceed to run the script
runScript(outputScript, ext, log, cwd)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Armstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Angstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def createMetadataImageStep(self):
#Create images.xmd metadata
fnImages = self._getOutputImgMd()
imgsXmd = md.MetaData()
posFiles = glob(self._getExtraPath('*.pos'))
for posFn in posFiles:
xmdFn = self._getExtraPath(replaceBaseExt(posFn, "xmd"))
if exists(xmdFn):
mdFn = md.MetaData(xmdFn)
mdPos = md.MetaData('particles@%s' % posFn)
mdPos.merge(mdFn)
#imgSet.appendFromMd(mdPos)
imgsXmd.unionAll(mdPos)
else:
self.warning(
"The coord file %s wasn't used to extract! Maybe you are extracting over a subset of micrographs"
% basename(posFn))
imgsXmd.write(fnImages)
def convertMask(img, outputDir):
""" Convert binary mask to a format read by Relion and truncate the
values between 0-1 values, due to Relion only support masks with this
values (0-1).
Params:
img: input image to be converted.
outputDir: where to put the converted file(s)
Return:
new file name of the mask.
"""
ih = em.ImageHandler()
imgFn = getImageLocation(img.getLocation())
newFn = join(outputDir, replaceBaseExt(imgFn, 'mrc'))
ih.truncateMask(imgFn, newFn)
return newFn
def _micrographsToEmx(emxData, micSet, emxDir, ctfSet=None, writeData=True):
""" Write a SetOfMicrograph as expected in EMX format (xml file)
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
"""
ih = ImageHandler()
for mic in micSet:
if writeData:
loc = mic.getLocation()
fnMicBase = replaceBaseExt(loc[1], 'mrc')
newLoc = join(emxDir, fnMicBase)
ih.convert(loc, newLoc)
mic.setLocation(NO_INDEX, fnMicBase)
if ctfSet:
mic.setCTF(ctfSet[mic.getObjId()])
emxMic = _micrographToEmx(mic)
emxData.addObject(emxMic)
def runTemplate(inputScript, ext, paramsDict, nummpis=1,
program=None, log=None, cwd=None):
""" This function will create a valid Spider script
by copying the template and replacing the values in dictionary.
After the new file is read, the Spider interpreter is invoked.
Usually the execution should be done where the results will
be left.
"""
if program is None:
program = Plugin.getProgram()
outputScript = replaceBaseExt(inputScript, ext)
if cwd is not None:
outputScript = join(cwd, outputScript)
# First write the script from the template with the substitutions
writeScript(inputScript, outputScript, paramsDict)
# Then proceed to run the script
runScript(outputScript, ext, program, nummpis, log, cwd)
def readSetOfCoordinates(outputDir, micSet, coordSet):
""" Read from Xmipp .pos files.
Params:
outputDir: the directory where the .pos files are.
It is also expected a file named: config.xmd
in this directory where the box size can be read.
micSet: the SetOfMicrographs to associate the .pos, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the config.xmd metadata
configfile = join(outputDir, 'config.xmd')
if exists(configfile):
md = xmipp.MetaData('properties@' + join(outputDir, 'config.xmd'))
boxSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE, md.firstObject())
coordSet.setBoxSize(boxSize)
for mic in micSet:
posFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'pos'))
readCoordinates(mic, posFile, coordSet, outputDir)
coordSet._xmippMd = String(outputDir)
def readSetOfCoordinates(outputDir, micSet, coordSet):
""" Read from Xmipp .pos files.
Params:
outputDir: the directory where the .pos files are.
It is also expected a file named: config.xmd
in this directory where the box size can be read.
micSet: the SetOfMicrographs to associate the .pos, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the config.xmd metadata
configfile = join(outputDir, 'config.xmd')
if exists(configfile):
md = xmipp.MetaData('properties@' + join(outputDir, 'config.xmd'))
boxSize = md.getValue(xmipp.MDL_PICKING_PARTICLE_SIZE, md.firstObject())
coordSet.setBoxSize(boxSize)
for mic in micSet:
posFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'pos'))
readCoordinates(mic, posFile, coordSet, outputDir)
coordSet._xmippMd = String(outputDir)
def writeSetOfCoordinates(posDir, coordSet, ismanual=True):
""" Write a pos file on metadata format for each micrograph
on the coordSet.
Params:
posDir: the directory where the .pos files will be written.
coordSet: the SetOfCoordinates that will be read.
"""
boxSize = coordSet.getBoxSize() or 100
# Create a dictionary with the pos filenames for each micrograph
posDict = {}
for mic in coordSet.iterMicrographs():
micName = mic.getFileName()
posFn = join(posDir, replaceBaseExt(micName, "pos"))
posDict[mic.getObjId()] = posFn
f = None
lastMicId = None
c = 0
for coord in coordSet.iterItems(orderBy='_micId'):
micId = coord.getMicId()
if micId != lastMicId:
# we need to close previous opened file
if f:
f.close()
#print "Micrograph %s (%d)" % (lastMicId, c)
c = 0
f = openMd(posDict[micId], ismanual=ismanual)
lastMicId = micId
c += 1
f.write(" %06d 1 %d %d %d %06d\n" % (coord.getObjId(),
coord.getX(), coord.getY(), 1,
micId))
if f:
f.close()
print "Micrograph %s (%d)" % (lastMicId, c)
state = 'Manual' if ismanual else 'Supervised'
# Write config.xmd metadata
configFn = join(posDir, 'config.xmd')
md = xmipp.MetaData()
# Write properties block
objId = md.addObject()
# micName = removeBaseExt(micName)
# md.setValue(xmipp.MDL_MICROGRAPH, str(micName), objId)
#md.setValue(xmipp.MDL_COLOR, int(-16776961), objId)
md.setValue(xmipp.MDL_PICKING_PARTICLE_SIZE, int(boxSize), objId)
md.setValue(xmipp.MDL_PICKING_STATE, state, objId)
md.write('properties@%s' % configFn)
# # Write filters block
# md = xmipp.MetaData()
# objId = md.addObject()
# md.setValue(xmipp.MDL_MACRO_CMD, 'Gaussian_Blur...', objId)
# md.setValue(xmipp.MDL_MACRO_CMD_ARGS, 'sigma=2', objId)
# md.write('filters@%s' % configFn, xmipp.MD_APPEND)
return posDict.values()
def writeSetOfCoordinates(posDir, coordSet):
""" Write a pos file on metadata format for each micrograph
on the coordSet.
Params:
posDir: the directory where the .pos files will be written.
coordSet: the SetOfCoordinates that will be read.
"""
boxSize = coordSet.getBoxSize() or 100
# Create a dictionary with the pos filenames for each micrograph
posDict = {}
for mic in coordSet.iterMicrographs():
micName = mic.getFileName()
posFn = join(posDir, replaceBaseExt(micName, "pos"))
posDict[mic.getObjId()] = posFn
f = None
lastMicId = None
c = 0
for coord in coordSet.iterItems(orderBy='_micId'):
micId = coord.getMicId()
if micId != lastMicId:
# we need to close previous opened file
if f:
f.close()
print "Micrograph %s (%d)" % (lastMicId, c)
c = 0
f = openMd(posDict[micId])
lastMicId = micId
c += 1
f.write(" %06d 1 %d %d %06d\n" % (coord.getObjId(),
coord.getX(), coord.getY(),
micId))
if f:
f.close()
print "Micrograph %s (%d)" % (lastMicId, c)
# Write config.xmd metadata
configFn = join(posDir, 'config.xmd')
md = xmipp.MetaData()
# Write properties block
objId = md.addObject()
# micName = removeBaseExt(micName)
# md.setValue(xmipp.MDL_MICROGRAPH, str(micName), objId)
#md.setValue(xmipp.MDL_COLOR, int(-16776961), objId)
md.setValue(xmipp.MDL_PICKING_PARTICLE_SIZE, int(boxSize), objId)
md.setValue(xmipp.MDL_PICKING_STATE, 'Manual', objId)
md.write('properties@%s' % configFn)
# # Write filters block
# md = xmipp.MetaData()
# objId = md.addObject()
# md.setValue(xmipp.MDL_MACRO_CMD, 'Gaussian_Blur...', objId)
# md.setValue(xmipp.MDL_MACRO_CMD_ARGS, 'sigma=2', objId)
# md.write('filters@%s' % configFn, xmipp.MD_APPEND)
return posDict.values()
def createOutputStep(self):
# Create the SetOfImages object on the database
#imgSet = XmippSetOfParticles(self._getPath('images.xmd'))
#Create images.xmd metadata
fnImages = self._getPath('images.xmd')
imgsXmd = xmipp.MetaData()
posFiles = glob(self._getExtraPath('*.pos'))
for posFn in posFiles:
xmdFn = self._getExtraPath(replaceBaseExt(posFn, "xmd"))
if exists(xmdFn):
md = xmipp.MetaData(xmdFn)
mdPos = xmipp.MetaData('particles@%s' % posFn)
mdPos.merge(md)
#imgSet.appendFromMd(mdPos)
imgsXmd.unionAll(mdPos)
else:
self.warning("The coord file %s wasn't used to extract! Maybe you are extracting over a subset of micrographs" % basename(posFn))
imgsXmd.write(fnImages)
# IF selected run xmipp_image_sort_by_statistics to add zscore info to images.xmd
if self.doSort:
args="-i %(fnImages)s --addToInput"
if self.rejectionMethod == REJECT_MAXZSCORE:
maxZscore = self.maxZscore.get()
args += " --zcut " + str(maxZscore)
elif self.rejectionMethod == REJECT_PERCENTAGE:
percentage = self.percentage.get()
args += " --percent " + str(percentage)
self.runJob("xmipp_image_sort_by_statistics", args % locals())
# Create output SetOfParticles
imgSet = self._createSetOfParticles()
imgSet.copyInfo(self.inputMics)
if self.doFlip:
# Check if self.inputMics are phase flipped.
if self.inputMics.isPhaseFlipped():
imgSet.setIsPhaseFlipped(False)
else:
imgSet.setIsPhaseFlipped(True)
#imgSet.setHasCTF(self.fnCTF is not None)
if self.downsampleType == OTHER:
imgSet.setSamplingRate(self.inputMics.getSamplingRate()*self.downFactor.get())
imgSet.setCoordinates(self.inputCoords)
# Create a temporary set to read from the metadata file
# and later create the good one with the coordinates
# properly set. We need this because the .update is not
# working in the mapper when new attributes are added.
imgSet.setHasCTF(self.ctfRelations.hasValue())
auxSet = SetOfParticles(filename=':memory:')
auxSet.copyInfo(imgSet)
readSetOfParticles(fnImages, auxSet)
# For each particle retrieve micId from SetOFCoordinates and set it on the CTFModel
for img in auxSet:
#FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)???
coord = self.inputCoords[img.getObjId()]
ctfModel = img.getCTF()
if ctfModel is not None:
ctfModel.setObjId(coord.getMicId())
##img.setCTF(ctfModel)####JM
img.setMicId(coord.getMicId())
img.setCoordinate(coord)
imgSet.append(img)
self._storeMethodsInfo(fnImages)
self._defineOutputs(outputParticles=imgSet)
self._defineSourceRelation(self.inputCoords, imgSet)
def _estimateCTF(self, micFn, micDir, micName):
""" Run the estimate CTF program """
localParams = self.__params.copy()
if self.doInitialCTF:
if self.ctfDict[micName] > 0:
localParams['defocusU'] = self.ctfDict[micName]
localParams['defocus_range'] = 0.01 * self.ctfDict[micName]
else:
localParams['defocusU'] = (self._params['maxDefocus'] +
self._params['minDefocus']) / 2
localParams['defocus_range'] = (self._params['maxDefocus'] -
self._params['minDefocus']) / 2
# Create micrograph dir under extra directory
makePath(micDir)
if not exists(micDir):
raise Exception("No created dir: %s " % micDir)
finalName = micFn
ctfDownFactor = self.ctfDownFactor.get()
downsampleList = [ctfDownFactor]
if self.doCTFAutoDownsampling:
downsampleList.append(ctfDownFactor + 1)
if ctfDownFactor >= 2:
downsampleList.append(ctfDownFactor - 1)
else:
if ctfDownFactor > 1:
downsampleList.append((ctfDownFactor + 1) / 2)
deleteTmp = ""
for downFactor in downsampleList:
# Downsample if necessary
if downFactor != 1:
#Replace extension by 'mrc' cause there are some formats that cannot be written (such as dm3)
finalName = self._getTmpPath(replaceBaseExt(micFn, 'mrc'))
self.runJob(
"xmipp_transform_downsample",
"-i %s -o %s --step %f --method fourier" %
(micFn, finalName, downFactor))
deleteTmp = finalName
# Update _params dictionary with mic and micDir
localParams['micFn'] = finalName
localParams['micDir'] = self._getFileName('prefix', micDir=micDir)
localParams['samplingRate'] = self.inputMics.getSamplingRate(
) * downFactor
# CTF estimation with Xmipp
try:
self.runJob(
self._program, self._args % localParams +
" --downSamplingPerformed %f" % downFactor)
# mdCTF = md.RowMetaData(self._getFileName('ctfparam', micDir=micDir))
except Exception:
break
# Check the quality of the estimation and reject it necessary
if self.evaluateSingleMicrograph(micFn, micDir):
break
if deleteTmp != "":
cleanPath(deleteTmp)
