def createMetadataImageStep(self):
mdUntilted = md.MetaData()
mdTilted = md.MetaData()
# for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = pwutils.removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in these micrographs
if pwutils.exists(fnMicU):
mdMicU = md.MetaData(fnMicU)
mdPosU = md.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = pwutils.removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = md.MetaData(fnMicT)
mdPosT = md.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadata (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the
# micrographs in the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
# create tmp set with all mics from coords set
coordMics = SetOfMicrographs(filename=':memory:')
coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())
for micU, micT in izip(
self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micU.cleanObjId()
micT.cleanObjId()
coordMics.append(micU)
coordMics.append(micT)
for mic in coordMics:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[pwutils.removeExt(mic.getMicName())] = micPos
# now match micDict and inputMics
if any(
pwutils.removeExt(mic.getMicName()) in micDict
for mic in self.inputMics):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception(
'Could not match input micrographs and coordinates '
'by micName.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' %
(micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def _postprocessImageRow(self, img, row):
""" Write the binary image to the final stack
and update the row imageName. """
if self._stackType > STACK_NONE:
rlnImageName = row.getValue('rlnImageName')
# backup the original name
row.setValue('rlnOriginalParticleName', rlnImageName)
if self._stackType == STACK_ONE:
self._count = getattr(self, '_count', 1)
index, stackName = (self._count, 'particles.mrcs')
self._count += 1
else: # STACK_MULT
baseName = pwutils.removeBaseExt(img.getFileName())
if baseName not in self._stackDict:
self._stackDict[baseName] = 0
index = self._stackDict[baseName] + 1
stackName = baseName + '.mrcs'
self._stackDict[baseName] = index
stackFn = self._getPath('Particles', stackName)
self._ih.convert(img, (index, stackFn))
# Store relative path in the star file
relStackFn = os.path.relpath(stackFn, self._getPath())
row.setValue('rlnImageName', locationToRelion(index, relStackFn))
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Prepare environment
from appion import Plugin
Plugin.getEnviron()
# Program to execute and it arguments
program = "python2"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(
inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
dogpicker = Plugin.getHome("ApDogPicker.py")
args = dogpicker + " " + args
self.runJob(program, args)
def runTomoSegmenTV(self, tomoFile):
tomoBaseName = removeBaseExt(tomoFile)
# Scale space
s2OutputFile = self._getExtraPath(tomoBaseName + S2 + MRC)
Plugin.runTomoSegmenTV(self, SCALE_SPACE,
self._getScaleSpaceCmd(tomoFile, s2OutputFile))
# Tensor voting
tVOutputFile = self._getExtraPath(tomoBaseName + TV + MRC)
Plugin.runTomoSegmenTV(
self, 'dtvoting',
self._getTensorVotingCmd(s2OutputFile, tVOutputFile))
# Surfaceness
surfOutputFile = self._getExtraPath(tomoBaseName + SURF + MRC)
Plugin.runTomoSegmenTV(self, 'surfaceness',
self._getSurfCmd(tVOutputFile, surfOutputFile))
# Tensor voting - second round (to fill potential gaps and increase the robustness of the surfaceness map)
tV2OutputFile = self._getExtraPath(tomoBaseName + TV2 + MRC)
Plugin.runTomoSegmenTV(
self, 'dtvoting',
self._getTensorVotingCmd(surfOutputFile,
tV2OutputFile,
isFirstRound=False))
# Saliency - second round (apply again the surfaceness program, but this time to produce the saliency)
salOutputFile = self._getExtraPath(tomoBaseName + FLT + MRC)
Plugin.runTomoSegmenTV(self, 'surfaceness',
self._getSalCmd(tV2OutputFile, salOutputFile))
self.tomoMaskListDelineated.append(salOutputFile)
# Remove intermediate files if requested
if not self.keepAllFiles.get():
self._removeIntermediateFiles(tomoFile)
def setCoords3D2Jsons(json_files, setCoords, mode="w"):
paths = []
groupIds = setCoords.aggregate(["MAX", "COUNT"], "_groupId", ["_groupId"])
groupIds = set([d['_groupId'] for d in groupIds])
emanIds = list(range(len(groupIds)))
dict_eman = dict(zip(groupIds, emanIds))
for json_file in json_files:
coords = []
for coor in setCoords.iterCoordinates():
tomoName = pwutils.removeBaseExt(coor.getVolume().getFileName())
if "__" in tomoName:
tomoName = '%s_info' % tomoName.split("__")[0]
else:
tomoName += "_info"
if tomoName in json_file:
coords.append([coor.getX(const.BOTTOM_LEFT_CORNER),
coor.getY(const.BOTTOM_LEFT_CORNER),
coor.getZ(const.BOTTOM_LEFT_CORNER),
"manual", 0.0, dict_eman[coor.getGroupId()]])
if coords:
coordDict = {"boxes_3d": coords,
"class_list": {}
}
for groupId in groupIds:
coordDict["class_list"]["%s" % dict_eman[groupId]] = {"boxsize": setCoords.getBoxSize(),
"name": "particles_%02d" % dict_eman[groupId]}
if mode == "w":
writeJson(coordDict, json_file)
paths.append(json_file)
elif mode == "a":
appendJson(coordDict, json_file)
paths.append(json_file)
return paths
def getOutputName(self, fn, key):
""" Give a key, append the mrc extension
and prefix the protocol working dir.
"""
template = pwutils.removeBaseExt(fn) + key + '.mrc'
return pwutils.join(self.getMicrographsDir(), template)
def readSetOfCoordinates(workDir,
micSet,
coordSet,
invertY=False,
newBoxer=False):
""" Read from Eman .json files.
Params:
workDir: where the Eman boxer output files are located.
micSet: the SetOfMicrographs to associate the .json, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
if newBoxer:
# read boxSize from info/project.json
jsonFnbase = pwutils.join(workDir, 'info', 'project.json')
jsonBoxDict = loadJson(jsonFnbase)
size = int(jsonBoxDict["global.boxsize"])
else:
# read boxSize from e2boxercache/base.json
jsonFnbase = pwutils.join(workDir, 'e2boxercache', 'base.json')
jsonBoxDict = loadJson(jsonFnbase)
size = int(jsonBoxDict["box_size"])
jsonFninfo = pwutils.join(workDir, 'info/')
for mic in micSet:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPosFn = ''.join(glob.glob(jsonFninfo + '*' + micBase +
'_info.json'))
readCoordinates(mic, micPosFn, coordSet, invertY)
coordSet.setBoxSize(size)
def _estimateCtfList(self, micList, *args, **kwargs):
""" Estimate several micrographs at once, probably a bit more
efficient. """
try:
micPath = self._getMicrographDir(micList[0])
if len(micList) > 1:
micPath += ('-%04d' % micList[-1].getObjId())
pwutils.makePath(micPath)
ih = emlib.image.ImageHandler()
for mic in micList:
micFn = mic.getFileName()
# We convert the input micrograph on demand if not in .mrc
downFactor = self.ctfDownFactor.get()
micFnMrc = os.path.join(micPath,
pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self._params['scannedPixelSize'] * downFactor
kwargs['scannedPixelSize'] = sps
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
program, args = self._gctfProgram.getCommand(**kwargs)
args += ' %s/*.mrc' % micPath
self.runJob(program, args) # , env=gctf.Plugin.getEnviron())
def _getFile(micBase, suffix):
return os.path.join(micPath, micBase + suffix)
for mic in micList:
micFn = mic.getFileName()
micBase = pwutils.removeBaseExt(micFn)
micFnMrc = _getFile(micBase, '.mrc')
# Let's clean the temporary mrc micrograph
pwutils.cleanPath(micFnMrc)
# move output from tmp to extra
micFnCtf = _getFile(micBase, self._gctfProgram.getExt())
micFnCtfLog = _getFile(micBase, '_gctf.log')
micFnCtfFit = _getFile(micBase, '_EPA.log')
micFnCtfOut = self._getPsdPath(micFn)
micFnCtfLogOut = self._getCtfOutPath(micFn)
micFnCtfFitOut = self._getCtfFitOutPath(micFn)
pwutils.moveFile(micFnCtf, micFnCtfOut)
pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
pwutils.cleanPath(micPath)
except:
print("ERROR: Gctf has failed on %s/*.mrc" % micPath)
import traceback
traceback.print_exc()
def getObjectInfo(self, tomo):
if self._mode == 'txt':
tomogramName = os.path.basename(tomo.getFileName())
tomogramName = os.path.splitext(tomogramName)[0]
filePath = os.path.join(self._path, tomogramName + ".txt")
elif self._mode == 'json':
tomogramName = os.path.basename(tomo.getFileName())
tomogramName = os.path.splitext(tomogramName)[0]
outFile = '*%s_info.json' % pwutils.removeBaseExt(
tomogramName.split("__")[0])
pattern = os.path.join(self._path, outFile)
files = glob.glob(pattern)
filePath = ''
if files:
filePath = files[0]
if not os.path.isfile(filePath):
return {
'key': tomogramName,
'parent': None,
'text': tomogramName,
'values': (tomo.count, "TODO"),
'tags': ("pending")
}
else:
return {
'key': tomogramName,
'parent': None,
'text': tomogramName,
'values': (tomo.count, "DONE"),
'tags': ("done")
}
def _checkNewInput(self):
# If continue from an stopped run, don't repeat what is done
if not self.checkedMics:
for fn in getFiles(self._getExtraPath()):
fn = removeBaseExt(fn)
if fn.startswith(self.FN_PREFIX):
self.checkedMics.update([self.getMicId(fn)])
self.processedMics.update([self.getMicId(fn)])
readyMics, self.streamClosed = getReadyMics(
self.inputCoordinates.get())
newMicsIds = readyMics.difference(self.checkedMics)
if newMicsIds:
self.checkedMics.update(newMicsIds)
inMics = self.getMainInput().getMicrographs()
newMics = [inMics[micId].clone() for micId in newMicsIds]
fDeps = self.insertNewCoorsSteps(newMics)
outputStep = self._getFirstJoinStep()
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def createOutputStep(self):
micSet = self.getInputMicrographs()
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
micSubSet = self._createSetOfMicrographs()
micSubSet.copyInfo(micSet)
for mic in self.getMicrographList():
micSubSet.append(mic)
self._defineOutputs(outputMicrographs=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
else:
# Clean up if previously created the outputMicrographs
if self.hasAttribute('outputMicrographs'):
self._deleteChild('outputMicrographs', self.outputMicrographs)
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [
template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet
]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet)
def create_movie_params(self, prot, updateIds):
for movie in self.iter_updated_set(prot.outputMovies):
movieId = movie.getObjId()
if movieId in self.movies: # this movie has been processed, skip
continue
movieFn = movie.getFileName()
if self.numberOfFrames is None:
self.numberOfFrames = movie.getNumberOfFrames()
images_path = self.find_ispyb_path(movieFn)
self.imageGenerator = ImageGenerator(self.project.path,
images_path,
bigThumb=True,
smallThumb=512)
acquisition = movie.getAcquisition()
self.movies[movieId] = {
'experimenttype': 'single particle',
'run_status': 'DataCollection Successful',
'imgdir': abspath(dirname(movieFn)),
'imgsuffix': pwutils.getExt(movieFn),
'file_template': pwutils.removeBaseExt(movieFn) + '#####' + pwutils.getExt(movieFn),
'file_location': abspath(dirname(movieFn)),
'filename': movieFn,
'n_passes': self.numberOfFrames,
'magnification': acquisition.getMagnification(),
'total_absorbed_dose': acquisition.getDoseInitial() + (acquisition.getDosePerFrame() * self.numberOfFrames),
'wavelength': self.convert_volts_to_debroglie_wavelength(acquisition.getVoltage())
}
self.dataCollection.update(self.movies[movieId])
self.update_from_metadata(Path(movieFn), self.movies[movieId])
updateIds.append(movieId)
def _postprocessParticleRow(self, part, partRow):
# Keep a map of the already converted files
self._stackDict = getattr(self, '_stackDict', {})
micBase = pwutils.removeBaseExt(part.getCoordinate().getMicName())
if micBase not in self._stackDict:
# Same convention without '_movie' suffix
newName = self._getExtraPath('%s.mrcs' % micBase)
self._linkOrConvertPart(part.getFileName(), newName)
self._stackDict[micBase] = newName
else:
newName = self._stackDict[micBase]
# The command will be launched from the working dir
# so, let's make the stack path relative to that
newPath = locationToRelion(part.getIndex(),
relpath(newName, self._getPath()))
partRow.setValue(md.RLN_IMAGE_NAME, newPath)
if part.hasAttribute('_rlnGroupName'):
partRow.setValue(md.RLN_MLMODEL_GROUP_NAME,
'%s' % part.getAttributeValue('_rlnGroupName'))
else:
partRow.setValue(md.RLN_MLMODEL_GROUP_NAME, '%s' % part.getMicId())
ctf = part.getCTF()
if ctf is not None and ctf.getPhaseShift():
partRow.setValue(md.RLN_CTF_PHASESHIFT, ctf.getPhaseShift())
def readCoordsFromMics(self, workingDir, micList, coordSet):
""" Parse back the output star files and populate the SetOfCoordinates.
"""
template = self._getExtraPath("%s_autopick.star")
starFiles = [template % pwutils.removeBaseExt(mic.getFileName())
for mic in micList]
readSetOfCoordinates(coordSet, starFiles, micList)
def readSetOfCoordinates(outputDir, micSet, coordSet):
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
inputCoords = args.extra
starFiles = [os.path.join(inputCoords,
pwutils.removeBaseExt(mic.getFileName())
+ '_autopick.star') for mic in micSet]
readSetOfCoordinates(coordSet, starFiles)
def createOutputStep(self):
# Lists of input tomograms and resized tomo masks are sorted by name to ensure that
# the relation between them is coherent
inTomoList = [tomo.clone() for tomo in self.inTomos.get()]
inTomoList.sort(key=self._sortTomoNames)
resizedFileList = sorted(self.resizedFileList)
inTomoMasksDir = getParentFolder(
self.inTomoMasks.get().getFirstItem().getFileName())
tomoMaskSet = SetOfTomoMasks.create(self._getPath(),
template='tomomasks%s.sqlite',
suffix='resized')
tomoMaskSet.copyInfo(self.inTomos.get())
counter = 1
for resizedFile, inTomo in zip(resizedFileList, inTomoList):
tomoMask = TomoMask()
tomoMask.copyInfo(inTomo)
tomoMask.setLocation((counter, resizedFile))
tomoMask.setVolName(inTomo.getFileName())
tomoMaskSet.append(tomoMask)
# Make a symbolic link to the corresponding annotation data file if necessary (in case
# of input set of annotated tomomasks)
annotationDataFile = join(inTomoMasksDir,
removeBaseExt(resizedFile) + '.txt')
if exists(annotationDataFile):
symlink(annotationDataFile,
self._getExtraPath(basename(annotationDataFile)))
counter += 1
self._defineOutputs(outputSetofTomoMasks=tomoMaskSet)
def _computeExtra(self, movie):
""" Compute thumbnail, PSD and plots. """
inputMovies = self.inputMovies.get()
movieFolder = self._getOutputMovieFolder(movie)
outMicFn = self._getMovieOutFn(movie, '.mrc')
if self.doComputeMicThumbnail:
self.computeThumbnail(outMicFn,
outputFn=self._getOutputMicThumbnail(movie))
if self.doComputePSD:
#fakeShiftsFn = self.writeZeroShifts(movie)
movieFn = movie.getFileName()
aveMicFn = os.path.join(
movieFolder,
pwutils.removeBaseExt(movieFn) + "_tmp.mrc")
self.averageMovie(movie,
movieFn,
aveMicFn,
binFactor=self.binFactor.get(),
dark=inputMovies.getDark(),
gain=inputMovies.getGain())
self.computePSDs(movie,
aveMicFn,
outMicFn,
outputFnCorrected=self._getPsdJpeg(movie))
self._saveAlignmentPlots(movie)
def _postprocessImageRow(self, img, row):
""" Write the binary image to the final stack
and update the row imageName. """
if self._stackType > STACK_NONE:
rlnImageName = row.getValue('rlnImageName')
# backup the original name
row.setValue('rlnOriginalParticleName', rlnImageName)
if self._stackType == STACK_ONE:
self._count = getattr(self, '_count', 1)
index, stackName = (self._count, 'particles.mrcs')
self._count += 1
else: # STACK_MULT
baseName = pwutils.removeBaseExt(img.getFileName())
if baseName not in self._stackDict:
self._stackDict[baseName] = 0
index = self._stackDict[baseName] + 1
stackName = baseName + '.mrcs'
self._stackDict[baseName] = index
stackFn = self._getPath('Particles', stackName)
self._ih.convert(img, (index, stackFn))
# Store relative path in the star file
relStackFn = os.path.relpath(stackFn, self._getPath())
row.setValue('rlnImageName', locationToRelion(index, relStackFn))
def createOutputStep(self):
tilt_series = self.tiltSeries.get()
out_tilt_series = self.getOutputSetOfTiltSeries(tilt_series)
json_paths = self._getJsonPaths()
for tilt_serie in tilt_series.iterItems():
out_tilt_serie = tomoObj.TiltSeries(tsId=tilt_serie.getTsId())
out_tilt_serie.copyInfo(tilt_serie)
out_tilt_series.append(out_tilt_serie)
item_basename = os.path.basename(os.path.dirname(tilt_serie.getFirstItem().getFileName())) +\
'-' + tilt_serie.getTsId() + '_info'
json_path = [
path for path in json_paths
if item_basename == pwutils.removeBaseExt(path)
]
params = loadJson(json_path[0])['tlt_params']
for idx, tiltImage in enumerate(tilt_serie.iterItems()):
out_tiltImage = tomoObj.TiltImage()
out_tiltImage.copyInfo(tiltImage, copyId=True)
out_tiltImage.setLocation(tiltImage.getLocation())
matrix = np.eye(3)
matrix[0, 0] = matrix[1,
1] = np.cos(np.deg2rad(params[idx][2]))
matrix[0, 1], matrix[1, 0] = np.sin(np.deg2rad(params[idx][2])), \
-np.sin(np.deg2rad(params[idx][2]))
out_tiltImage.tiltAngleAxis = Float(params[idx][4])
out_tiltImage.setTiltAngle(params[idx][3])
matrix[0, 2], matrix[1, 2] = params[idx][0], params[idx][1]
transform = data.Transform()
transform.setMatrix(matrix)
out_tiltImage.setTransform(transform)
out_tilt_serie.append(out_tiltImage)
self._defineOutputs(alignedTiltSeries=out_tilt_series)
self._defineSourceRelation(self.tiltSeries, out_tilt_series)
def _getMicCTF(mic):
micName = mic.getMicName()
micBase = removeBaseExt(mic.getFileName())
# see if the base name of this mic is contained in other base names
micConflicts = [mc for mc in inputMicBases if micBase in mc and micBase != mc]
if micConflicts:
self.warning('WARNING: Micrograph base name "%s" conflicts with micrograph(s) "%s". '
'Will try to find a unique match...' % (micBase, '", "'.join(micConflicts)))
# check which matching file only matches with this mic and not its conflicts
goodFnMatches = [f for f in files if micBase in f and not any(c in f for c in micConflicts)]
for goodFnMatch in goodFnMatches:
try:
micCtf = ci.importCTF(mic, goodFnMatch)
self.warning("WARNING: Assigned file %s to micrograph %s." % (goodFnMatch, micBase))
return micCtf
except Exception as ex:
self.warning("WARNING: Can't import ctf for micrograph %s from file %s"
% (micBase, goodFnMatch))
continue
else:
return None
else:
for fileName, fileId in self.iterFiles():
if (fileId == mic.getObjId() or
micBase in fileName or micName in fileName):
return ci.importCTF(mic, fileName)
return None
def getMatchingMic(self, coordFile, fileId):
""" Given a coordinates file check if there is a micrograph
that this files matches.
"""
micSet = self.inputMicrographs.get()
if fileId is None:
coordBase = pwutils.removeBaseExt(coordFile)
for mic in micSet:
micBase = pwutils.removeBaseExt(mic.getFileName())
# temporal use of in
if coordBase in micBase or micBase in coordBase:
return mic
return None
else:
return micSet[fileId]
def processTiltImageStep(self, tsId, tiltImageId, *args):
tiltImageM = self._tsDict.getTi(tsId, tiltImageId)
workingFolder = self.__getTiltImageMWorkingFolder(tiltImageM)
pw.utils.makePath(workingFolder)
self._processTiltImageM(workingFolder, tiltImageM, *args)
if self._doSplitEvenOdd():
baseName = removeBaseExt(tiltImageM.getFileName())
evenName = abspath(self._getExtraPath(baseName + '_avg_' + EVEN))
oddName = abspath(self._getExtraPath(baseName + '_avg_' + ODD))
alignedFrameStack = self._getExtraPath(baseName +
'_aligned_movie.mrcs')
# Get even/odd xmd files
args = '--img %s ' % abspath(alignedFrameStack)
args += '--type frames '
args += '-o %s ' % (evenName + '.xmd')
args += '-e %s ' % (oddName + '.xmd')
args += '--sum_frames '
self.runJob('xmipp_image_odd_even', args)
# Store the corresponding tsImM to use its data later in the even/odd TS
self.tsMList.append(tiltImageM)
# Update even and odd average lists
self.evenAvgFrameList.append(evenName + '_aligned.mrc')
self.oddAvgFrameList.append(oddName + '_aligned.mrc')
pw.utils.cleanPath(alignedFrameStack)
tiFn, tiFnDW = self._getOutputTiltImagePaths(tiltImageM)
if not os.path.exists(tiFn):
raise Exception("Expected output file '%s' not produced!" % tiFn)
if not pw.utils.envVarOn('SCIPION_DEBUG_NOCLEAN'):
pw.utils.cleanPath(workingFolder)
def _restimateCTF(self, ctfId):
""" Run Gctf with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = out
pwutils.cleanPath(psdFile)
try:
self.runJob(self._getProgram(), self._args % self._params)
except:
print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
pwutils.cleanPattern(micFnMrc)
def createOutputStep(self):
micSet = self.getInputMicrographs()
outputCoordinatesName = 'outputCoordinates'
outputSuffix = ''
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
outputSuffix = '_subset'
outputCoordinatesName = 'outputCoordinatesSubset'
micSubSet = self._createSetOfMicrographs(suffix=outputSuffix)
micSubSet.copyInfo(micSet)
# Use previously written star file for reading the subset of micrographs,
for row in md.iterRows(self._getPath('input_micrographs.star')):
mic = micSet[row.getValue('rlnImageId')]
micSubSet.append(mic)
self._defineOutputs(outputMicrographsSubset=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(**{outputCoordinatesName: coordSet})
self._defineSourceRelation(self.getInputMicrographsPointer(),
coordSet)
def detectCarbonCloseness(self, coordinates):
self.scoreCarbon = []
for tomoName in self.tomoNames:
idt = self.tomo_vesicles[tomoName]["volId"]
tomo = self.tomos[idt].clone()
projFile = self.projectTomo(tomo)
inputTomoPathMetadataFname = self._getTmpPath("inputTomo.xmd")
tomo_md = createMetaDataFromPattern(projFile)
tomo_md.write(inputTomoPathMetadataFname)
coordList = self.generateCoordList(tomo, coordinates)
self.writeTomoCoordinates(tomo, coordList,
self._getTomoPos(projFile))
args = '-i %s -c %s -o %s -b %d' \
% (inputTomoPathMetadataFname, self._getExtraPath('inputCoords'),
self._getExtraPath('outputCoords'), coordinates.getBoxSize())
self.runJob('xmipp_deep_micrograph_cleaner',
args,
env=Plugin.getTensorFlowEnviron())
baseName = pwutils.removeBaseExt(projFile)
outFile = self._getExtraPath('outputCoords', baseName + ".pos")
posMd = readPosCoordinates(outFile)
posMd.addLabel(md.MDL_ITEM_ID)
for objId in posMd:
if posMd.getValue(md.MDL_ENABLED, objId) == 0:
posMd.setValue(md.MDL_ENABLED, 1, objId)
coord = rowToCoordinate(rowFromMd(posMd, objId))
self.scoreCarbon.append([
posMd.getValue(md.MDL_ITEM_ID, objId),
coord._xmipp_goodRegionScore.get()
])
pwutils.cleanPath(projFile)
def createOutputStep(self):
micSet = self.getInputMicrographs()
outputCoordinatesName = 'outputCoordinates'
outputSuffix = ''
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
outputSuffix = '_subset'
outputCoordinatesName = 'outputCoordinatesSubset'
micSubSet = self._createSetOfMicrographs(suffix=outputSuffix)
micSubSet.copyInfo(micSet)
# Use previously written star file for reading the subset of micrographs,
for row in md.iterRows(self._getPath('input_micrographs.star')):
mic = micSet[row.getValue('rlnImageId')]
micSubSet.append(mic)
self._defineOutputs(outputMicrographsSubset=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [
template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet
]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(**{outputCoordinatesName: coordSet})
self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet)
def getOutputName(self, fn, key):
""" Give a key, append the mrc extension
and prefix the protocol working dir.
"""
template = pwutils.removeBaseExt(fn) + key + '.mrc'
return pwutils.join(self.getMicrographsDir(), template)
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the
# micrographs in the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
# create tmp set with all mics from coords set
coordMics = SetOfMicrographs(filename=':memory:')
coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())
for micU, micT in izip(self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micU.cleanObjId()
micT.cleanObjId()
coordMics.append(micU)
coordMics.append(micT)
for mic in coordMics:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[pwutils.removeExt(mic.getMicName())] = micPos
# now match micDict and inputMics
if any(pwutils.removeExt(mic.getMicName()) in micDict for mic in self.inputMics):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception('Could not match input micrographs and coordinates '
'by micName.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' % (micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def _getMicExtra(self, mic, suffix):
""" Return a file in extra direction with root of micFn.
:param mic: input mic object
:param suffix: file extension
"""
return self._getExtraPath(
pwutils.removeBaseExt(os.path.basename(mic.getFileName())) + '_' +
suffix)
def createOutputStep(self):
micSet = self.getInputMicrographs()
coordSet = self._createSetOfCoordinates(micSet)
coordSet.setBoxSize(self.getInputReferences().getDim()[0])
starFiles = [self._getExtraPath(pwutils.removeBaseExt(mic.getFileName()) + "_autopick.star") for mic in micSet]
readSetOfCoordinates(coordSet, starFiles)
self._defineOutputs(outputCoordinates=coordSet)
def _newMic(mic):
micBase = pwutils.removeBaseExt(mic.getFileName())
ctfFn = self._getCtfLocalOutPath(micBase)
self._rowCounter = 0
if os.path.exists(ctfFn):
self._rowList = [row.clone() for row in md.iterRows(ctfFn)]
else:
self._rowList = None
def _preprocessMovieRow(self, movie, movieRow):
# 'movie' suffix in newName is required for Relion
micBase = pwutils.removeBaseExt(movie.getMicName())
newName = self._getExtraPath('%s_movie.mrcs' % micBase)
self._linkOrConvertMovie(movie.getFileName(), newName)
# The command will be launched from the working dir
# so, let's make the movie path relative to that
movie.setFileName(relpath(newName, self._getPath()))
def _insertMicStep(mic):
localDeps = [firstStepId]
fnLast = mic.getFileName()
micName = pwutils.removeBaseExt(mic.getFileName())
def getMicTmp(suffix):
return self._getTmpPath(micName + suffix)
# Create a list with micrographs operations (programs in xmipp) and
# the required command line parameters (except input/ouput files)
micOps = []
# Check if it is required to downsample your micrographs
downFactor = self.downFactor.get()
if self.notOne(downFactor):
fnDownsampled = getMicTmp("_downsampled.xmp")
args = "-i %s -o %s --step %f --method fourier"
micOps.append(('xmipp_transform_downsample',
args % (fnLast, fnDownsampled, downFactor)))
fnLast = fnDownsampled
if self.doRemoveDust:
fnNoDust = getMicTmp("_noDust.xmp")
args = " -i %s -o %s --bad_pixels outliers %f"
micOps.append(('xmipp_transform_filter',
args % (fnLast, fnNoDust, self.thresholdDust)))
fnLast = fnNoDust
if self._useCTF() and self.ctfDict[mic] is not None:
# We need to write a Xmipp ctfparam file
# to perform the phase flip on the micrograph
fnCTF = self._getTmpPath("%s.ctfParam" % micName)
mic.setCTF(self.ctfDict[mic])
micrographToCTFParam(mic, fnCTF)
# Insert step to flip micrograph
if self.doFlip:
fnFlipped = getMicTmp('_flipped.xmp')
args = " -i %s -o %s --ctf %s --sampling %f"
micOps.append(
('xmipp_ctf_phase_flip', args %
(fnLast, fnFlipped, fnCTF, self._getNewSampling())))
fnLast = fnFlipped
else:
fnCTF = None
# Actually extract
deps.append(
self._insertFunctionStep('extractParticlesStep',
mic.getObjId(),
micName,
fnCTF,
fnLast,
micOps,
self.doInvert.get(),
self._getNormalizeArgs(),
self.doBorders.get(),
prerequisites=localDeps))
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the
# micrographs in the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
for micU, micT in izip(
self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micBaseU = pwutils.removeBaseExt(micU.getFileName())
micPosU = self._getExtraPath(micBaseU + ".pos")
micDict[pwutils.removeExt(micU.getMicName())] = micPosU
micBaseT = pwutils.removeBaseExt(micT.getFileName())
micPosT = self._getExtraPath(micBaseT + ".pos")
micDict[pwutils.removeExt(micT.getMicName())] = micPosT
# now match micDict and other mics (in self.ctfDict)
if any(
pwutils.removeExt(mic.getMicName()) in micDict
for mic in self.ctfDict):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception(
'Could not match input micrographs and coordinates '
'by micName.')
for mic in self.ctfDict:
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' %
(micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def projectTomo(self, tomo):
outFile = pwutils.removeBaseExt(tomo.getFileName()) + '_projected.mrc'
ih = ImageHandler()
outProjection = ih.createImage()
tomoData = np.squeeze(ih.read(tomo.getFileName()).getData())
projection = np.sum(tomoData, axis=0)
outProjection.setData(projection)
ih.write(outProjection, self._getExtraPath(outFile))
return self._getExtraPath(outFile)
def readCoordsFromMics(self, workingDir, micList, coordSet):
""" Parse back the output star files and populate the SetOfCoordinates.
"""
template = self._getExtraPath("%s_autopick.star")
starFiles = [
template % pwutils.removeBaseExt(mic.getFileName())
for mic in micList
]
readSetOfCoordinates(coordSet, starFiles, micList)
def writeXmippCoords(self):
""" Write the SetOfCoordinates as expected by Xmipp
to be display with its GUI.
"""
micSet = self.getInputMicrographs()
coordSet = self._createSetOfCoordinates(micSet)
coordSet.setBoxSize(self.getInputReferences().getDim()[0])
starFiles = [self._getExtraPath(pwutils.removeBaseExt(mic.getFileName()) + "_autopick.star") for mic in micSet]
readSetOfCoordinates(coordSet, starFiles)
return self._writeXmippCoords(coordSet)
def readSetOfCoordinates(workingDir, micSet, coordSet):
""" Read from coordinates from Gempicker .box files.
For a micrograph: mic1.mrc, the expected coordinate file is: mic1.box
Params:
workingDir: where the Gempicker .box output files are located.
micSet: the input SetOfMicrographs
coordSet: the SetOfCoordinates that will be populated.
"""
for mic in micSet:
micBase = pwutils.removeBaseExt(mic.getFileName())
fnCoords = join(workingDir, 'pik_box', micBase + '.box')
readCoordinates(mic, fnCoords, coordSet)
def _insertAllSteps(self):
self._setupBasicProperties()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps, run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
fnLast = mic.getFileName()
micName = pwutils.removeBaseExt(mic.getFileName())
def getMicTmp(suffix):
return self._getTmpPath(micName + suffix)
# Create a list with micrographs operations (programs in xmipp) and
# the required command line parameters (except input/ouput files)
micOps = []
# Check if it is required to downsample your micrographs
downFactor = self.downFactor.get()
if self.notOne(downFactor):
fnDownsampled = getMicTmp("_downsampled.xmp")
args = "-i %s -o %s --step %f --method fourier"
micOps.append(('xmipp_transform_downsample',
args % (fnLast, fnDownsampled, downFactor)))
fnLast = fnDownsampled
if self.doRemoveDust:
fnNoDust = getMicTmp("_noDust.xmp")
args = " -i %s -o %s --bad_pixels outliers %f"
micOps.append(('xmipp_transform_filter',
args % (fnLast, fnNoDust, self.thresholdDust)))
fnLast = fnNoDust
# TODO: implement CTF
# Actually extract
deps.append(self._insertFunctionStep('extractParticlesStep',
mic.getObjId(), micName,
None, fnLast, micOps,
self.doInvert.get(),
self._getNormalizeArgs(),
self.doBorders.get(),
prerequisites=localDeps))
metaDeps = self._insertFunctionStep('createMetadataImageStep',
prerequisites=deps)
# Insert step to create output objects
self._insertFunctionStep('createOutputStep',
prerequisites=[metaDeps], wait=False)
def createCtfModelStep(self):
inputSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(inputSet)
for particle in inputSet:
coord = particle.getCoordinate()
if coord is None:
continue
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
xNew, yNew = (xCoor * self.scale, yCoor * self.scale)
else:
xNew, yNew = (x * self.scale, y * self.scale)
micBase = pwutils.removeBaseExt(coord.getMicName())
for key in self.matchingMics:
micKey = pwutils.removeBaseExt(key.getFileName())
if micBase in micKey:
# micName from mic and micName from coord may be different
ctfFn = pwutils.join(self._getExtraPath(micKey),
micKey + '_local.star')
if pwutils.exists(ctfFn):
mdFn = md.MetaData(ctfFn)
for row in md.iterRows(mdFn):
coordX = row.getValue(md.RLN_IMAGE_COORD_X)
coordY = row.getValue(md.RLN_IMAGE_COORD_Y)
if (int(xNew), int(yNew)) == (coordX, coordY):
newPart = particle.clone()
rowToCtfModel(row, newPart.getCTF())
partSet.append(newPart)
self._defineOutputs(outputParticles=partSet)
self._defineTransformRelation(inputSet, partSet)
def main():
parser = argparse.ArgumentParser(prog='Scipion Convert')
parser.add_argument('--coordinates', help='Convert coordinates', action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder')
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
#print 'converting coordinates ...'
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
#print 'from eman2 to xmipp...'
from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'dogpicker':
if toType == 'xmipp':
#print 'from dogpicker to xmipp...'
from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'relion':
if toType == 'xmipp':
#print 'from relion to xmipp...'
inputCoords = args.extra
starFiles = [os.path.join(inputCoords, pwutils.removeBaseExt(mic.getFileName()) + '_autopick.star')
for mic in micSet]
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
readSetOfCoordinates(coordSet, starFiles)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
def writeSetOfCoordinates(coordDir, coordSet, micsSet):
""" Write a star file on metadata format for each micrograph
on the coordSet.
Params:
coordDir: the directory where the .star files will be written.
coordSet: the SetOfCoordinates that will be read.
micsSet: the SetOfMicrographs that will be read.
"""
header = """
data_
loop_
_rlnCoordinateX #1
_rlnCoordinateY #2
"""
# Create a dictionary with the pos filenames for each micrograph
posDict = {}
for mic in micsSet:
micBase = pwutils.removeBaseExt(mic.getFileName())
posDict[mic.getObjId()] = pwutils.join(coordDir, micBase,
micBase + '_coords.star')
f = None
lastMicId = None
# Iterate only once over the whole SetOfCoordinates, but ordering by
# micrograph Id, so we can detect when there are coordinates from a
# new micrographs to write the new star file
for coord in coordSet.iterItems(orderBy='_micId'):
micId = coord.getMicId()
if micId != lastMicId: # Detect there is a new micrograph
if f: # we need to close previous opened file
f.close()
f = open(posDict[micId], 'w')
f.write(header)
lastMicId = micId
f.write("%d %d\n" % coord.getPosition())
if f:
f.close()
def create_movie_params(self, prot, allParams):
for movie in self.iter_updated_set(prot.outputMovies):
movieFn = movie.getFileName()
if self.numberOfFrames is None:
self.numberOfFrames = movie.getNumberOfFrames()
images_path = self.find_ispyb_path(movieFn)
self.imageGenerator = ImageGenerator(self.project.path,
images_path,
smallThumb=512)
movieId = movie.getObjId()
allParams[movieId] = {
'id': self.allIds.get(movieId, None),
'imgdir': dirname(movieFn),
'imgprefix': pwutils.removeBaseExt(movieFn),
'imgsuffix': pwutils.getExt(movieFn),
'file_template': movieFn,
'n_images': self.numberOfFrames
}
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
self.runJob(program, args)
def _pickMicrographList(self, micList, args):
""" Pick several micrographs at once, probably a bit more efficient."""
micFnList = []
for mic in micList:
micFn = mic.getFileName()
micFnList.append(micFn)
# The coordinates conversion is done for each micrograph
# and not in convertInputStep, this is needed for streaming
badCoords = self.inputBadCoords.get()
if self.exclusive and badCoords:
fnCoords = os.path.join(self.getMicrographsDir(), '%s_rubbish.star'
% pwutils.removeBaseExt(micFn))
writeMicCoords(mic, badCoords.iterCoordinates(mic), fnCoords)
# We convert the input micrograph on demand if not in .mrc
runGautomatch(micFnList,
self._getReferencesFn(),
self.getMicrographsDir(),
args,
env=self._getEnviron(),
runJob=self.runJob)
def getOutputFn(self):
""" Returns the scaled output file name
"""
outputFnBase = removeBaseExt(self.inputVolFn)
return self._getExtraPath(outputFnBase) + '_scaled.mrc'
def _getMicPos(self, mic):
""" Return the corresponding .pos file for a given micrograph. """
micBase = pwutils.removeBaseExt(mic.getFileName())
return self._getExtraPath(micBase + ".coords.star")
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale, yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
def refineCtfStep(self):
self._defineValues()
self._prepareCommand()
for mic in self.matchingMics:
micName = mic.getFileName()
micBase = pwutils.removeBaseExt(micName)
micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
outMic = pwutils.join(micDirTmp, pwutils.replaceBaseExt(micName, 'mrc'))
micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
micFnOut = self._getCtfOutPath(micDirTmp)
micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')
# Update _params dictionary
self._params['micFn'] = outMic
self._params['gctfOut'] = micFnOut
if self.useInputCtf and self.ctfRelations.get():
# get input CTFs from a mic
ctfs = self.ctfRelations.get()
micKey = mic.getMicName() if self.hasMicName else mic.getObjId()
for ctf in ctfs:
ctfMicName = ctf.getMicrograph().getMicName()
ctfMicId = ctf.getMicrograph().getObjId()
if micKey == ctfMicName or micKey == ctfMicId:
# add CTF refine options
self._params.update({'refine_input_ctf': 1,
'defU_init': ctf.getDefocusU(),
'defV_init': ctf.getDefocusV(),
'defA_init': ctf.getDefocusAngle(),
'B_init': self.bfactor.get()
})
self._args += "--refine_input_ctf %d " % self._params['refine_input_ctf']
self._args += "--defU_init %f " % self._params['defU_init']
self._args += "--defV_init %f " % self._params['defV_init']
self._args += "--defA_init %f " % self._params['defA_init']
self._args += "--B_init %f " % self._params['B_init']
self._args += "--defU_err %f " % self.defUerr.get()
self._args += "--defV_err %f " % self.defVerr.get()
self._args += "--defA_err %f " % self.defAerr.get()
self._args += "--B_err %f " % self.Berr.get()
break
# final args
self._args += "--do_validation %d " % (1 if self.doValidate else 0)
self._args += "%(micFn)s "
self._args += "> %(gctfOut)s"
try:
self.runJob(self._getProgram(), self._args % self._params,
env=self._getEnviron())
except:
print("ERROR: Gctf has failed for micrograph %s" % outMic)
# move results from tmp to extra folder
micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
psdFile = self._getPsdPath(micDir)
ctfOutFile = self._getCtfOutPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
ctflocalFile = self._getCtfLocalPath(micDir, micBase)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnOut, ctfOutFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.moveFile(micFnLocalCtf, ctflocalFile)
# Let's clean the temporary micrographs
pwutils.cleanPath(outMic)
pwutils.cleanPath(micDirTmp)
pwutils.cleanPath(self.matchingMics.getFileName())
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
