def extractParticles(self):
samplingRateCoord = self.inputCoordinates.get().getSamplingRate()
samplingRateTomo = self.getInputTomograms().getFirstItem(
).getSamplingRate()
for tomo in self.tomoFiles:
args = '%s ' % os.path.abspath(tomo)
args += "--coords %s --boxsize %i" % (pwutils.replaceBaseExt(
tomo, 'coords'), self.boxSize.get())
if self.doInvert:
args += ' --invert'
if self.doNormalize:
args += ' --normproc %s' % self.getEnumText('normproc')
args += ' --cshrink %d' % (samplingRateCoord / samplingRateTomo)
program = emantomo.Plugin.getProgram('e2spt_boxer_old.py')
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
moveFile(
self._getExtraPath(os.path.join('sptboxer_01',
'basename.hdf')),
self._getExtraPath(pwutils.replaceBaseExt(tomo, 'hdf')))
cleanPath(self._getExtraPath("sptboxer_01"))
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
#Replace extension by 'mrc' cause there are some formats that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
self.runJob("xmipp_transform_downsample",
"-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor),
env=xmipp3.getEnviron())
self._params['scannedPixelSize'] = self.inputMicrographs.get().getScannedPixelSize() * downFactor
else:
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Prepare environment
from appion import Plugin
Plugin.getEnviron()
# Program to execute and it arguments
program = "python2"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(
inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
dogpicker = Plugin.getHome("ApDogPicker.py")
args = dogpicker + " " + args
self.runJob(program, args)
def _restimateCTF(self, ctfId):
""" Run Gctf with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = out
pwutils.cleanPath(psdFile)
try:
self.runJob(self._getProgram(), self._args % self._params)
except:
print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
pwutils.cleanPattern(micFnMrc)
def _estimateCTF(self, mic, *args):
""" Run ctftilt with required parameters """
micFn = mic.getFileName()
micDir = self._getTmpPath('mic_%04d' % mic.getObjId())
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize(
)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' because there are some formats
# that cannot be written (such as dm3)
em.ImageHandler().scaleFourier(micFn, micFnMrc, downFactor)
self._params[
'scannedPixelSize'] = scannedPixelSize * downFactor
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
micFnMrc = self._getTmpPath(
pwutils.replaceBaseExt(micFn, "mrc"))
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
except Exception as ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
try:
program, args = self._getCommand(
micFn=micFnMrc,
ctftiltOut=self._getCtfOutPath(micDir),
ctftiltPSD=self._getPsdPath(micDir))
self.runJob(program, args)
except Exception as ex:
print >> sys.stderr, "ctftilt has failed with micrograph %s" % micFnMrc
# Let's notify that this micrograph have been processed
# just creating an empty file at the end (after success or failure)
open(doneFile, 'w')
# Let's clean the temporary mrc micrographs
pwutils.cleanPath(micFnMrc)
def convertPdbStep(self):
vol1 = self.vol1.get()
pdbFn = self._getPdbFileName()
self.outFile = self._getVolName()
if getExt(pdbFn) == ".cif":
pdbFn2 = replaceBaseExt(pdbFn, 'pdb')
cifToPdb(pdbFn, pdbFn2)
pdbFn = pdbFn2
samplingR = vol1.getSamplingRate()
size = vol1.getDim()
ccp4header = headers.Ccp4Header(vol1.getFileName(), readHeader=True)
self.shifts = ccp4header.getOrigin()
# convert pdb to volume with the size and origin of the input volume
args = ' -i %s --sampling %f -o %s --size %d %d %d --orig %d %d %d' % \
(pdbFn, samplingR, removeExt(self.outFile), size[2], size[1], size[0], self.shifts[0]/samplingR,
self.shifts[1]/samplingR, self.shifts[2]/samplingR)
program = "xmipp_volume_from_pdb"
self.runJob(program, args)
volpdbmrc = "%s.mrc" % removeExt(self.outFile)
# convert volume from pdb to .mrc in order to store the origin in the mrc header
args2 = ' -i %s -o %s -t vol' % (self.outFile, volpdbmrc)
program2 = "xmipp_image_convert"
self.runJob(program2, args2)
# write origin in mrc header of volume from pdb
ccp4headerOut = headers.Ccp4Header(volpdbmrc, readHeader=True)
ccp4headerOut.setOrigin(self.shifts)
def _restimateCTF(self, ctfId):
""" Run ctffind3 with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = out
self._params['ctffindPSD'] = psdFile
pwutils.cleanPath(psdFile)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def __getMicFile(self, mic, ext):
""" Return a filename based on the micrograph.
The filename will be located in the extra folder and with
the given extension.
"""
return self._getExtraPath(
pwutils.replaceBaseExt(mic.getFileName(), ext))
def _convert(self, image):
imageFn = image.getFileName()
if self.outputDir is None:
return imageFn
ext = pwutils.getExt(imageFn)[1:]
if ext in self.extensions:
finalExt = ext
convertFunc = pwutils.createAbsLink
else:
finalExt = self.extensions[0]
convertFunc = self._ih.convert
if self.useBaseName:
newName = pwutils.replaceBaseExt(image.getFileName(), finalExt)
else:
newName = "%s_%06d.%s" % (self._prefix, image.getObjId(), finalExt)
newPath = os.path.join(self.outputDir, newName)
convertFunc(os.path.abspath(imageFn), newPath)
# If there is a rootDir defined, we should return the path relative
# to that location, probably to be used from the star file
if self.rootDir is not None:
newPath = os.path.relpath(newPath, self.rootDir)
return newPath
def convertInputStep(self, micsId, refsId):
""" This step will take of the convertions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrc' format
Mask: either converted ('.tif' format) or generated a circular one
"""
ih = em.ImageHandler()
micDir = self._getTmpPath('micrographs')
pwutils.makePath(micDir)
for mic in self.getInputMicrographs():
# Create micrograph folder
micName = mic.getFileName()
# If micrographs are in .mrc format just link it
# otherwise convert them
outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(mic, outMic)
refDir = self._getTmpPath('templates')
pwutils.makePath(refDir)
# We will always convert the templates, since
# they can be in an stack and link will not be possible sometimes
inputRefs = self.inputReferences.get()
for i, ref in enumerate(inputRefs):
outRef = join(refDir, 'ref%02d.mrc' % (i+1))
ih.convert(ref, outRef)
self.createInputMasks(inputRefs)
def _getCoordsStarFile(mic):
micName = mic.getMicName()
if not micName in micDict:
return None
micFn = micDict[micName]
return self._getExtraPath(
pwutils.replaceBaseExt(micFn, 'coords.star'))
def test_readDM4(self):
""" Check we can read dm4 files (using EMAN)
"""
micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4')
ih = emlib.image.ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (7676, 7420, 1, 1)
self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)
# We could even convert to an mrc file:
outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')
outFn = join('/tmp', outSuffix)
print("Converting: \n%s -> %s" % (micFn, outFn))
ih.convert(micFn, outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
# Check dimensions are still the same:
self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)
# Clean up tmp files
pwutils.cleanPath(outFn)
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameter just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
imgStarTmp = self._getTmpPath('movie_particles.star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStarTmp))
writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(),
alignType=imgSet.getAlignment(),
extraLabels=MOVIE_EXTRA_LABELS)
mdImg = md.MetaData(imgStarTmp)
# replace mdColumn from *.stk to *.mrcs as Relion2 requires
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
from convert import relionToLocation, locationToRelion
for objId in mdImg:
index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId))
if not imgPath.endswith('mrcs'):
newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs')
newPath = self._getTmpPath(newName)
newLoc = locationToRelion(index, newPath)
if not exists(newPath):
pwutils.createLink(imgPath, newPath)
mdImg.setValue(mdColumn, newLoc, objId)
mdImg.write(imgStar)
def _estimateCtfList(self, micList, *args, **kwargs):
""" Estimate several micrographs at once, probably a bit more
efficient. """
try:
micPath = self._getMicrographDir(micList[0])
if len(micList) > 1:
micPath += ('-%04d' % micList[-1].getObjId())
pwutils.makePath(micPath)
ih = emlib.image.ImageHandler()
for mic in micList:
micFn = mic.getFileName()
# We convert the input micrograph on demand if not in .mrc
downFactor = self.ctfDownFactor.get()
micFnMrc = os.path.join(micPath,
pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self._params['scannedPixelSize'] * downFactor
kwargs['scannedPixelSize'] = sps
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
program, args = self._gctfProgram.getCommand(**kwargs)
args += ' %s/*.mrc' % micPath
self.runJob(program, args) # , env=gctf.Plugin.getEnviron())
def _getFile(micBase, suffix):
return os.path.join(micPath, micBase + suffix)
for mic in micList:
micFn = mic.getFileName()
micBase = pwutils.removeBaseExt(micFn)
micFnMrc = _getFile(micBase, '.mrc')
# Let's clean the temporary mrc micrograph
pwutils.cleanPath(micFnMrc)
# move output from tmp to extra
micFnCtf = _getFile(micBase, self._gctfProgram.getExt())
micFnCtfLog = _getFile(micBase, '_gctf.log')
micFnCtfFit = _getFile(micBase, '_EPA.log')
micFnCtfOut = self._getPsdPath(micFn)
micFnCtfLogOut = self._getCtfOutPath(micFn)
micFnCtfFitOut = self._getCtfFitOutPath(micFn)
pwutils.moveFile(micFnCtf, micFnCtfOut)
pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
pwutils.cleanPath(micPath)
except:
print("ERROR: Gctf has failed on %s/*.mrc" % micPath)
import traceback
traceback.print_exc()
def _insertAllSteps(self):
atomsFn = self.getInputPdb().getFileName()
# Define some outputs filenames
self.imgsFn = self._getExtraPath('images.xmd')
self.modesFn = self._getExtraPath('modes.xmd')
self.structureEM = self.inputModes.get().getPdb().getPseudoAtoms()
if self.structureEM:
self.atomsFn = self._getExtraPath(basename(atomsFn))
copyFile(atomsFn, self.atomsFn)
else:
localFn = self._getExtraPath(
replaceBaseExt(basename(atomsFn), 'pdb'))
cifToPdb(atomsFn, localFn)
self.atomsFn = self._getExtraPath(basename(localFn))
self._insertFunctionStep('convertInputStep', atomsFn)
if self.copyDeformations.empty(): #ONLY FOR DEBUGGING
self._insertFunctionStep("performNmaStep", self.atomsFn,
self.modesFn)
else:
# TODO: for debugging and testing it will be useful to copy the deformations
# metadata file, not just the deformation.txt file
self._insertFunctionStep('copyDeformationsStep',
self.copyDeformations.get())
self._insertFunctionStep('createOutputStep')
def joinSetOfParticlesStep(self, mode):
#Create images.xmd metadata joining from different .stk
fnImages = self._getExtraPath("particles_%s.xmd" % mode)
imgsXmd = MD.MetaData()
for i in range(len(self.inputCoordinatesTrue)):
posFiles = glob(
self._getExtraPath(
self.CONSENSUS_COOR_PATH_TEMPLATE % (mode, i), '*.pos'))
for posFn in posFiles:
xmdFn = self._getExtraPath(
self.CONSENSUS_PARTS_PATH_TEMPLATE % (mode, i),
pwutils.replaceBaseExt(posFn, "xmd"))
if os.path.exists(xmdFn):
mdFn = MD.MetaData(xmdFn)
mdPos = MD.MetaData('particles@%s' % posFn)
mdPos.merge(mdFn)
imgsXmd.unionAll(mdPos)
else:
self.warning(
"The coord file %s wasn't used for extraction! " %
os.path.basename(posFn))
self.warning(
"Maybe you are extracting over a subset of micrographs"
)
imgsXmd.write(fnImages)
def convertInputStep(self, micsId, refsId):
""" This step will take of the convertions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrc' format
Mask: either converted ('.tif' format) or generated a circular one
"""
ih = em.ImageHandler()
micDir = self._getTmpPath('micrographs')
pwutils.makePath(micDir)
for mic in self.getInputMicrographs():
# Create micrograph folder
micName = mic.getFileName()
# If micrographs are in .mrc format just link it
# otherwise convert them
outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(mic, outMic)
refDir = self._getTmpPath('templates')
pwutils.makePath(refDir)
# We will always convert the templates, since
# they can be in an stack and link will not be possible sometimes
inputRefs = self.inputReferences.get()
for i, ref in enumerate(inputRefs):
outRef = join(refDir, 'ref%02d.mrc' % (i + 1))
ih.convert(ref, outRef)
self.createInputMasks(inputRefs)
def writeSetOfCoordinates3D(self):
tomoList = []
for tomo in self.getInputTomograms():
tomoList.append(tomo.clone())
for tomo in tomoList:
coordDict = []
self.coordsFileName = self._getExtraPath(
pwutils.replaceBaseExt(tomo.getFileName(), 'coords'))
with open(self.coordsFileName, "w") as out:
coords = self.inputCoordinates.get()
for coord3D in coords.iterCoordinates(volume=tomo):
if os.path.basename(
tomo.getFileName()) == os.path.basename(
coord3D.getVolName()):
out.write("%d\t%d\t%d\n" %
(coord3D.getX(const.BOTTOM_LEFT_CORNER),
coord3D.getY(const.BOTTOM_LEFT_CORNER),
coord3D.getZ(const.BOTTOM_LEFT_CORNER)))
newCoord = coord3D.clone()
newCoord.setVolume(coord3D.getVolume())
coordDict.append(newCoord)
if coordDict:
self.lines.append(coordDict)
self.tomoFiles.append(tomo.getFileName())
self.samplingRateTomo = tomo.getSamplingRate()
def convertPdbStep(self):
""" Although is not mandatory, usually is used by the protocol to
register the resulting outputs in the database.
"""
pdbFn = self._getPdbFileName()
outFile = removeExt(self._getVolName())
if getExt(pdbFn) == ".cif":
pdbFn2 = replaceBaseExt(pdbFn, 'pdb')
cifToPdb(pdbFn, pdbFn2)
pdbFn = pdbFn2
args = '-i %s --sampling %f -o %s' % (pdbFn, self.sampling.get(),
outFile)
if self.centerPdb:
args += ' --centerPDB'
if self.setSize:
args += ' --size'
if self.size.hasValue():
args += ' %d' % self.size.get()
self.info("Input file: " + pdbFn)
self.info("Output file: " + outFile)
program = "xmipp_volume_from_pdb"
self.runJob(program, args)
def createOutputStep(self):
micSet = self.getInputMicrographs()
ih = em.ImageHandler()
coordSet = self._createSetOfCoordinates(micSet)
if self.boxSize == 0:
coordSet.setBoxSize(self.inputReferences.get().getDim()[0])
else:
coordSet.setBoxSize(self.boxSize.get())
for mic in micSet:
fnCoords = pwutils.replaceBaseExt(mic.getFileName(), 'txt')
fn2parse = self._getExtraPath('pik_coord', fnCoords)
print fn2parse
xdim, _, _, _ = ih.getDimensions(mic)
#xdim, ydim, _ = em.getDimensions(micSet)
#print xdim, ydim
with open(fn2parse,"r") as source:
source.readline() # Skip first line
for line in source:
tokens = line.split()
coord = em.Coordinate()
coord.setPosition(xdim-int(tokens[3]), int(tokens[2]))
coord.setMicrograph(mic)
coordSet.append(coord)
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(micSet, coordSet)
def createOutputStep(self):
micSet = self.getInputMicrographs()
ih = em.ImageHandler()
coordSet = self._createSetOfCoordinates(micSet)
if self.boxSize == 0:
coordSet.setBoxSize(self.inputReferences.get().getDim()[0])
else:
coordSet.setBoxSize(self.boxSize.get())
for mic in micSet:
fnCoords = pwutils.replaceBaseExt(mic.getFileName(), 'txt')
fn2parse = self._getExtraPath('pik_coord', fnCoords)
print fn2parse
xdim, _, _, _ = ih.getDimensions(mic)
#xdim, ydim, _ = em.getDimensions(micSet)
#print xdim, ydim
with open(fn2parse, "r") as source:
source.readline() # Skip first line
for line in source:
tokens = line.split()
coord = em.Coordinate()
coord.setPosition(xdim - int(tokens[3]), int(tokens[2]))
coord.setMicrograph(mic)
coordSet.append(coord)
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(micSet, coordSet)
def update_ctf_params(self, prot, updateIds):
for ctf in self.iter_updated_set(prot.outputCTF):
micId = ctf.getObjId()
if self.motion_corrections.get(micId, None) is not None:
if micId in self.ctfs and 'status' in self.ctfs[micId]: # skip if we already have align info
continue
micFn = ctf.getMicrograph().getFileName()
psdName = pwutils.replaceBaseExt(micFn, 'psd.jpeg')
psdFn = ctf.getPsdFile()
psdPng = self.imageGenerator.generate_image(psdFn, psdName)
if micId not in self.ctfs:
self.ctfs[micId] = {}
self.ctfs[micId].update({
'estimatedDefocus': ((ctf.getDefocusV() + ctf.getDefocusU())/2),
'astigmatism': ctf.getDefocusRatio(),
'estimatedResolution': ctf.getResolution(),
'astigmatismAngle': ctf.getDefocusAngle(),
'fftTheoreticalFullPath': psdPng,
'programs': getattr(prot, '_label', lambda x: None),
'status': 1
})
print('%d has new ctf info' % micId)
updateIds.append(micId)
def convertToMrc(fn):
""" Convert from a format that is not read by Relion
to mrc format.
"""
newFn = os.path.join(outputDir, replaceBaseExt(fn, 'mrc'))
ih.convert(fn, newFn)
return newFn
def readSetOfCoordinates(outputDir, micSet, coordSet):
""" Read from Bsoft .star files.
Params:
outputDir: the directory where the .star files are.
micSet: the SetOfMicrographs to associate the .star, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
boxSize = 100
for mic in micSet:
outputFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'star'))
if exists(outputFile):
posMd = md.MetaData(outputFile)
for objId in posMd:
coordRow = rowFromMd(posMd, objId)
coord = rowToCoordinate(coordRow)
boxSize = 2 * float(coordRow.getValue("particle.x_origin", 50))
coord.setMicrograph(mic)
coord.setX(coord.getX())
coord.setY(coord.getY())
coordSet.append(coord)
# Add an unique ID that will be propagated to particles
# posMd.setValue(md.MDL_PARTICLE_ID, long(coord.getObjId()), objId)
# reading origin.x value and converting to particle
# size, can change, we take last value
coordSet.setBoxSize(boxSize)
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize(
)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' because there are some formats
# that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
args = "-i %s -o %s --step %f --method fourier" % (
micFn, micFnMrc, downFactor)
self.runJob("xmipp_transform_downsample",
args,
env=xmipp3.getEnviron())
self._params[
'scannedPixelSize'] = scannedPixelSize * downFactor
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
micFnMrc = self._getTmpPath(
pwutils.replaceBaseExt(micFn, "mrc"))
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
def _createOutputMovie(self, movie):
n = movie.getNumberOfFrames()
newMovie = movie.clone()
movieName = pwutils.replaceBaseExt(movie.getFileName(), 'tif')
newMovie.setFileName(self._getExtraPath(movieName))
newMovie.setAlignment(
MovieAlignment(first=1, last=n, xshifts=[0] * n, yshifts=[0] * n))
return newMovie
def readSetOfCoordinates(workDir, micSet, coordSet):
""" Read coordinates from cisTEM .plt files.
:param workDir: input folder with coord files
:param micSet: input set of mics
:param coordSet: output set of coords
"""
for mic in micSet:
micCoordFn = join(workDir, replaceBaseExt(mic.getFileName(), 'plt'))
readCoordinates(mic, micCoordFn, coordSet)
def getMicrographMetadata(self, micName):
metadataName = pwutils.replaceBaseExt(micName, 'star')
metadataPath = os.path.join(os.path.dirname(self._starFile), "motion",
metadataName)
print("Loading metadata file {}".format(metadataPath))
if not os.path.isfile(metadataPath):
print("Metadata for {} not found".format(micName))
return (False)
return (metadataPath)
def writeSetOfSubtomograms(self, subtomoSet, subtomosStar, isPyseg=False):
currentTomo = ''
MRC = 'mrc'
ih = ImageHandler()
tomoTable = self._createStarTomoTable(isPyseg)
tmpDir = pwutils.getParentFolder(subtomosStar)
for subtomo in subtomoSet:
if pwutils.getExt(subtomo.getFileName().replace(':' + MRC,
'')) != '.' + MRC:
mrcDir = join(tmpDir,
pwutils.removeBaseExt(subtomo.getVolName()))
if currentTomo != subtomo.getVolName():
mkdir(mrcDir)
mrcFile = join(
mrcDir, pwutils.replaceBaseExt(subtomo.getFileName(), MRC))
ih.convert(subtomo.getFileName(), mrcFile)
angles, shifts = self._getTransformInfoFromSubtomo(subtomo)
magn = subtomo.getAcquisition().getMagnification()
rlnMicrographName = subtomo.getVolName()
rlnCoordinateX = subtomo.getCoordinate3D().getX(BOTTOM_LEFT_CORNER)
rlnCoordinateY = subtomo.getCoordinate3D().getY(BOTTOM_LEFT_CORNER)
rlnCoordinateZ = subtomo.getCoordinate3D().getZ(BOTTOM_LEFT_CORNER)
rlnImageName = subtomo.getFileName().replace(':' + MRC, '')
rlnCtfImage = self._getCTFFileFromSubtomo(subtomo)
rlnMagnification = magn if magn else 10000 #64000
rlnDetectorPixelSize = subtomo.getSamplingRate()
rlnAngleRot = angles[0]
rlnAngleTilt = angles[1]
rlnAnglePsi = angles[2]
rlnOriginX = shifts[0]
rlnOriginY = shifts[1]
rlnOriginZ = shifts[2]
rlnTiltPrior = subtomo._tiltPriorAngle.get() if hasattr(
subtomo, '_tiltPriorAngle') else rlnAngleTilt
rlnTiltPsi = subtomo._psiPriorAngle.get() if hasattr(
subtomo, '_psiPriorAngle') else rlnAnglePsi
# Add row to the table which will be used to generate the STAR file
fieldsToAdd = [
rlnMicrographName, rlnCoordinateX, rlnCoordinateY,
rlnCoordinateZ, rlnImageName, rlnCtfImage, rlnMagnification,
rlnDetectorPixelSize, rlnAngleRot, rlnAngleTilt, rlnTiltPrior,
rlnAnglePsi, rlnTiltPsi, rlnOriginX, rlnOriginY, rlnOriginZ
]
if isPyseg:
fieldsToAdd = [
rlnMicrographName, rlnCoordinateX, rlnCoordinateY,
rlnCoordinateZ, rlnImageName, rlnCtfImage, rlnAngleRot,
rlnAngleTilt, rlnAnglePsi, rlnOriginX, rlnOriginY,
rlnOriginZ
]
tomoTable.addRow(*fieldsToAdd)
# Write the STAR file
tomoTable.write(subtomosStar)
def runGautomatch(cls,
micNameList,
refStack,
workDir,
extraArgs,
env=None,
runJob=None):
""" Run Gautomatch with the given parameters.
If micrographs are not .mrc, they will be converted.
If runJob=None, it will use pwutils.runJob.
"""
args = ''
ih = ImageHandler()
for micName in micNameList:
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workDir,
pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createAbsLink(os.path.abspath(micName), outMic)
else:
ih.convert(micName, outMic)
args += ' %s/*.mrc' % workDir
if refStack is not None:
args += ' -T %s' % refStack
args += ' %s' % extraArgs
environ = env if env is not None else cls.getEnviron()
if runJob is None:
pwutils.runJob(None, cls.getProgram(), args, env=environ)
else:
runJob(cls.getProgram(), args, env=environ)
for micName in micNameList:
outMic = os.path.join(workDir,
pwutils.replaceBaseExt(micName, 'mrc'))
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def update_ctf_params(self, prot, allParams):
for ctf in self.iter_updated_set(prot.outputCTF):
micFn = ctf.getMicrograph().getFileName()
psdName = pwutils.replaceBaseExt(micFn, 'psd.png')
psdFn = ctf.getPsdFile()
psdPng = self.imageGenerator.generate_image(psdFn, psdName)
allParams[ctf.getObjId()].update({
'min_defocus': ctf.getDefocusU(),
'max_defocus': ctf.getDefocusV(),
'amount_astigmatism': ctf.getDefocusRatio()
})
def writeSetOfCoordinatesXmipp(posDir, coordSet, ismanual=True, scale=1):
""" Write a pos file on metadata format for each micrograph
on the coordSet.
Params:
posDir: the directory where the .pos files will be written.
coordSet: the SetOfCoordinates that will be read."""
boxSize = coordSet.getBoxSize() or 100
state = 'Manual' if ismanual else 'Supervised'
# Create a dictionary with the pos filenames for each micrograph
posDict = {}
for mic in coordSet.iterMicrographs():
micIndex, micFileName = mic.getLocation()
micName = os.path.basename(micFileName)
if micIndex != NO_INDEX:
micName = '%06d_at_%s' % (micIndex, micName)
posFn = pwutils.join(posDir, pwutils.replaceBaseExt(micName, "pos"))
posDict[mic.getObjId()] = posFn
f = None
lastMicId = None
c = 0
for coord in coordSet.iterItems(orderBy='_micId'):
micId = coord.getMicId()
if micId != lastMicId:
# we need to close previous opened file
if f:
f.close()
c = 0
f = openMd(posDict[micId], state)
lastMicId = micId
c += 1
if scale != 1:
x = coord.getX() * scale
y = coord.getY() * scale
else:
x = coord.getX()
y = coord.getY()
f.write(" %06d 1 %d %d %d %06d\n"
% (coord.getObjId(), x, y, 1, micId))
if f:
f.close()
# Write config.xmd metadata
configFn = pwutils.join(posDir, 'config.xmd')
writeCoordsConfig(configFn, int(boxSize), state)
return posDict.values()
def update_ctf_params(self, prot, allParams):
for ctf in self.iter_updated_set(prot.outputCTF):
micFn = ctf.getMicrograph().getFileName()
psdName = pwutils.replaceBaseExt(micFn, 'psd.png')
psdFn = ctf.getPsdFile()
psdPng = self.imageGenerator.generate_image(psdFn, psdName)
allParams[ctf.getObjId()].update({
'min_defocus': ctf.getDefocusU(),
'max_defocus': ctf.getDefocusV(),
'amount_astigmatism': ctf.getDefocusRatio()
})
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' because there are some formats
# that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
args = "-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor)
self.runJob("xmipp_transform_downsample",
args, env=xmipp3.getEnviron())
self._params['scannedPixelSize'] = scannedPixelSize * downFactor
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
def _estimateCTF(self, micFn, micDir, micName):
""" Run Gctf with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
ih = em.ImageHandler()
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(
pwutils.removeBaseExt(micFn) + '_EPA.log')
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self.inputMicrographs.get().getScannedPixelSize(
) * downFactor
self._params['scannedPixelSize'] = sps
else:
if ih.existsLocation(micFn):
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = self._getCtfOutPath(micDir)
print >> sys.stdout, "Updated the paths dictionary with %s %s %s" % (
micFnMrc, micDir, self._getCtfOutPath)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
def _pickMicrograph(self, mic, radius):
micFn = mic.getFileName()
micDir = self._getMicDir(micFn)
fnMicBase = pwutils.replaceBaseExt(micFn, 'mrc')
fnMicCfg = pwutils.replaceBaseExt(micFn, 'cfg')
fnMicFull = os.path.join(micDir, fnMicBase)
fnPosBase = self._getMicPosFn(micFn)
# Convert micrographs to mrc (uint8) as required by ETHAN program
ih = ImageHandler()
ih.convert(micFn, fnMicFull, md.DT_UCHAR)
# Create a configuration file to be used by ETHAN with the parameters
# selected by the user
self.writeConfigFile(os.path.join(micDir, fnMicCfg))
# Run ethan program with the required arguments
program = self.getProgram()
args = "%s %s %s %s" % (radius, fnMicBase, fnPosBase, fnMicCfg)
self.runJob(program, args, cwd=micDir)
# Clean temporary micrograph
pwutils.cleanPath(fnMicFull)
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
self.runJob(program, args)
def _preprocessMicrographRow(self, img, imgRow):
# Temporarly convert the few micrographs to tmp and make sure
# they are in 'mrc' format
# Get basename and replace extension by 'mrc'
newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc')
newPath = self._getExtraPath(newName)
# If the micrographs are in 'mrc' format just create a link
# if not, convert to 'mrc'
if img.getFileName().endswith('mrc'):
pwutils.createLink(img.getFileName(), newPath)
else:
self._ih.convert(img, newPath)
# The command will be launched from the working dir
# so, let's make the micrograph path relative to that
img.setFileName(os.path.join('extra', newName))
def runGempicker(micName, workingDir, useGPU, args, log=None):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
refDir = join(workingDir, 'templates')
maskSchDir = join(workingDir, 'maskSch')
args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir,
refDir, maskSchDir)
# Run Gempicker:
for mode in [0, 1]:
pwutils.runJob(log, getProgram(useGPU), args + ' --mode=%d' % mode,
env=getEnviron())
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def _micrographsToEmx(emxData, micSet, emxDir, ctfSet=None, writeData=True):
""" Write a SetOfMicrograph as expected in EMX format (xml file)
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
"""
ih = ImageHandler()
for mic in micSet:
if writeData:
loc = mic.getLocation()
fnMicBase = pwutils.replaceBaseExt(loc[1], 'mrc')
newLoc = join(emxDir, fnMicBase)
ih.convert(loc, newLoc)
mic.setLocation(NO_INDEX, fnMicBase)
if ctfSet:
mic.setCTF(ctfSet[mic.getObjId()])
emxMic = _micrographToEmx(mic)
emxData.addObject(emxMic)
def alignParticlesStep(self, innerRadius, outerRadius):
""" Execute the pairwise.msa script to alignm the particles.
"""
particles = self.inputParticles.get()
xdim = particles.getDimensions()[0]
self._params.update({
'[idim-header]': xdim,
'[cg-option]': self.cgOption.get(),
'[inner-rad]': innerRadius,
'[outer-rad]': outerRadius, # convert radius to diameter
'[search-range]': self.searchRange.get(),
'[step-size]': self.stepSize.get(),
'[selection_list]': self._params['particlesSel'],
'[unaligned_image]': self._params['particles'] + '@******',
'[nummps]': self.numberOfThreads.get(),
})
copy1Script = getScript('mda/center1.msa')
newScript = pwutils.replaceBaseExt(copy1Script, self.getExt())
pwutils.copyFile(copy1Script, self._getPath(newScript))
self.runTemplate(self.getScript(), self.getExt(), self._params)
def _getMicPosFn(self, micFn):
return pwutils.replaceBaseExt(micFn, 'txt')
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = em.Particle()
# Let's create a dict with the names of the mic in Relion star files
# and also create a set with all star files to iterate them once
starSet = set()
micPathDict = {}
for mic in micList:
starSet.add(self._micCoordStarDict[mic.getObjId()])
micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc')
micFile = os.path.join('extra', micName)
micPathDict[micFile] = mic
prevMicFile = None
mic = None
for partStarFn in starSet:
for row in md.iterRows(self._getPath(partStarFn),
sortByLabel=md.RLN_MICROGRAPH_NAME):
micFile = row.getValue(md.RLN_MICROGRAPH_NAME)
if micFile != prevMicFile: # Load some stuff when new mic
if prevMicFile is not None: # cleanup previous mic
# Release the list of coordinates for this micrograph
# since it will not be longer needed
del self.coordDict[mic.getObjId()]
mic = micPathDict[micFile]
coordDict = {self._getPos(coord): coord
for coord in self.coordDict[mic.getObjId()]}
posSet = set() # Set of (x, y) pairs to avoid duplicates
prevMicFile = micFile
pos = (row.getValue(md.RLN_IMAGE_COORD_X),
row.getValue(md.RLN_IMAGE_COORD_Y))
if pos in posSet:
print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
coord = None
else:
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME))
p.setLocation(idx, self._getPath(fn[2:]))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
outputParts.append(p)
posSet.add(pos)
# Clean up the last mic if necessary
if mic is not None:
del self.coordDict[mic.getObjId()]
def __getMicFile(self, mic, ext):
""" Return a filename based on the micrograph.
The filename will be located in the extra folder and with
the given extension.
"""
return self._getExtraPath(pwutils.replaceBaseExt(mic.getFileName(),ext))
def _getVolName(self):
return self._getExtraPath(replaceBaseExt(self._getPdbFileName(), "vol"))
def _getMicStarFile(self, mic):
return self._getExtraPath(pwutils.replaceBaseExt(mic.getFileName(), 'star'))
def refineCtfStep(self):
self._defineValues()
self._prepareCommand()
for mic in self.matchingMics:
micName = mic.getFileName()
micBase = pwutils.removeBaseExt(micName)
micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
outMic = pwutils.join(micDirTmp, pwutils.replaceBaseExt(micName, 'mrc'))
micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
micFnOut = self._getCtfOutPath(micDirTmp)
micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')
# Update _params dictionary
self._params['micFn'] = outMic
self._params['gctfOut'] = micFnOut
if self.useInputCtf and self.ctfRelations.get():
# get input CTFs from a mic
ctfs = self.ctfRelations.get()
micKey = mic.getMicName() if self.hasMicName else mic.getObjId()
for ctf in ctfs:
ctfMicName = ctf.getMicrograph().getMicName()
ctfMicId = ctf.getMicrograph().getObjId()
if micKey == ctfMicName or micKey == ctfMicId:
# add CTF refine options
self._params.update({'refine_input_ctf': 1,
'defU_init': ctf.getDefocusU(),
'defV_init': ctf.getDefocusV(),
'defA_init': ctf.getDefocusAngle(),
'B_init': self.bfactor.get()
})
self._args += "--refine_input_ctf %d " % self._params['refine_input_ctf']
self._args += "--defU_init %f " % self._params['defU_init']
self._args += "--defV_init %f " % self._params['defV_init']
self._args += "--defA_init %f " % self._params['defA_init']
self._args += "--B_init %f " % self._params['B_init']
self._args += "--defU_err %f " % self.defUerr.get()
self._args += "--defV_err %f " % self.defVerr.get()
self._args += "--defA_err %f " % self.defAerr.get()
self._args += "--B_err %f " % self.Berr.get()
break
# final args
self._args += "--do_validation %d " % (1 if self.doValidate else 0)
self._args += "%(micFn)s "
self._args += "> %(gctfOut)s"
try:
self.runJob(self._getProgram(), self._args % self._params,
env=self._getEnviron())
except:
print("ERROR: Gctf has failed for micrograph %s" % outMic)
# move results from tmp to extra folder
micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
psdFile = self._getPsdPath(micDir)
ctfOutFile = self._getCtfOutPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
ctflocalFile = self._getCtfLocalPath(micDir, micBase)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnOut, ctfOutFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.moveFile(micFnLocalCtf, ctflocalFile)
# Let's clean the temporary micrographs
pwutils.cleanPath(outMic)
pwutils.cleanPath(micDirTmp)
pwutils.cleanPath(self.matchingMics.getFileName())
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale, yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
