def _preprocessImageRow30(self, img, imgRow):
from .convert_deprecated import setupCTF, copyOrLinkFileName
if self._imgPath is not None:
copyOrLinkFileName(imgRow, self._imgPath,
self.protocol._getExtraPath())
setupCTF(imgRow, self.protocol.samplingRate.get())
if self._micIdOrName:
micId = imgRow.get('rlnMicrographId', None)
micName = imgRow.get('rlnMicrographName', None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath(
'fake_micrograph%6d' % micId)
mic.setFileName(micName)
mic.setMicName(os.path.basename(micName))
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
# Update the row to set a MDL_MICROGRAPH_ID
imgRow['rlnMicrographId'] = int(mic.getObjId())
def getMicList(start=1, end=3):
""" Mocks a mic dict"""
newMicDict = {}
for index in range(start, end + 1):
mic = Micrograph()
mic.setMicName("mic%s" % index)
newMicDict[index] = mic
return newMicDict
def getCTFList(start=1, end=3):
""" Mocks a CTF dict"""
newCtfDict = {}
for index in range(start, end + 1):
ctf = CTFModel()
mic = Micrograph()
mic.setMicName("mic%s" % index)
ctf.setMicrograph(mic)
newCtfDict[mic.getMicName()] = ctf
return newCtfDict
def appendMicsFromTomogram(self, output, tomo):
self.info("Creating micrographs for %s" % tomo.getFileName())
# Load the tomogram
ih = ImageHandler()
img = ih.read(tomo.getFileName())
data = img.getData()
# For each slice
for index in range(0, len(data), self.slicesGap.get()):
self.debug("Creating micrograph for slice %s" % index)
micName = tomoSliceToMicName(tomo, index)
outputMicName = self._getExtraPath(micName)
outputMicName = replaceExt(outputMicName, "mrc")
slice = data[index]
micImg = ImageHandler()
micImg._img.setData(slice)
micImg.write(micImg._img, outputMicName)
# Create the micrograph metadata object
newMic = Micrograph()
newMic.setFileName(outputMicName)
newMic.setMicName(micName)
newMic.setSamplingRate(tomo.getSamplingRate())
# Append it
output.append(newMic)
def testCtfConsensus1(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
# create one fake micrographs image
projSize = 32
img = emlib.Image()
img.setDataType(emlib.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 4000 + 10 * i
defocusV = 4000 + i
defocusAngle = i * 10
resolution = 2
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle,
resolution, psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
ctfSet1.write()
micSet.write()
# import micrograph set
args = {
'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
self.launchProtocol(protCTF1)
# launch CTF consensus protocol
protCtfConsensus = self.newProtocol(XmippProtCTFConsensus)
protCtfConsensus.inputCTF.set(protCTF1.outputCTF)
protCtfConsensus.setObjLabel('ctf consensus')
self.launchProtocol(protCtfConsensus)
self.checkOutputSize(protCtfConsensus)
ctf0 = protCtfConsensus.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 4010)
def _preprocessImageRow30(self, img, imgRow):
self.preprocess_success = False
#Create a link or copy the particle binary files (*.mrcs). If the file already exists, does nothing
self.copyOrLinkBinary(imgRow,
'rlnImageName',
self._imgPath,
self.protocol._getExtraPath(),
copyFiles=self.protocol.copyBinaries.get())
setupCTF(imgRow, self.acquisitionDict['samplingRate'])
movieId = imgRow.get('rlnMicrographId', None)
movieName = imgRow.get('rlnMicrographName', None)
#Import the associated movie (*.mrcs)
if self.movieSet is not None and imgRow.get('rlnMicrographName',
None) is not None:
# Check which is the key to identify micrographs (id or name)
if movieId is not None:
movieKey = movieId
else:
movieKey = movieName
# First time I found this micrograph (either by id or name)
if movieKey not in self._importedMovies:
self.copyOrLinkBinary(
imgRow,
'rlnMicrographName',
self._imgPath,
self.protocol._getExtraPath(),
copyFiles=self.protocol.copyBinaries.get())
movieName = imgRow.get('rlnMicrographName', None)
movie = Movie()
movie.setObjId(movieId)
movie.setFileName(movieName)
movie.setMicName(movieName)
if self._importAlignments:
alignment = self.getMicrographAlignment(movie)
if alignment:
movie.setAlignment(alignment)
self.movieSet.append(movie)
# Update dict with new movie
self._movieDict[movieKey] = movie
self._importedMovies.add(movieKey)
else:
self.preprocess_success = False
else:
self.movieSet.append(movie)
# Update dict with new movie
self._movieDict[movieKey] = movie
self._importedMovies.add(movieKey)
if self.micSet is not None:
imgRow['rlnMicrographName'] = self.fixMicName(
imgRow['rlnMicrographName'])
micName = imgRow.get('rlnMicrographName', None)
micId = imgRow.get('rlnMicrographId', None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
# First time I found this micrograph (either by id or name)
if micKey not in self._importedMicrographs:
self.copyOrLinkBinary(
imgRow,
'rlnMicrographName',
self._imgPath,
self.protocol._getExtraPath(),
copyFiles=self.protocol.copyBinaries.get())
micName = imgRow.get('rlnMicrographName', None)
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath(
'fake_micrograph%6d' % micId)
mic.setFileName(micName)
mic.setMicName(movieName)
ctf = rowToCtfModel(imgRow)
ctf.setMicrograph(mic)
self.ctfSet.append(ctf)
mic.setCTF(rowToCtfModel(imgRow))
self.micSet.append(mic)
self._micDict[os.path.basename(movieName)] = mic
self._importedMicrographs.add(micKey)
else:
mic = self._micDict.get(os.path.basename(movieName))
movieName = mic.getMicName()
# Update the row to set a MDL_MICROGRAPH_ID
imgRow['rlnMicrographId'] = int(mic.getObjId())
imgRow['rlnMicrographName'] = movieName
img.setCTF(rowToCtfModel(imgRow))
self.preprocess_success = True
def createOutputStep(self):
oddSet = self._createSetOfMovies(suffix='odd')
evenSet = self._createSetOfMovies(suffix='even')
for movie in self.inputMovies.get():
fnMovie = movie.getFileName()
fnMovieOddMrc = self._getExtraPath(
pwutils.removeExt(basename(fnMovie)) + "_odd.mrc")
fnMovieEvenMrc = self._getExtraPath(
pwutils.removeExt(basename(fnMovie)) + "_even.mrc")
imgOutOdd = Movie()
imgOutEven = Movie()
imgOutOdd.setFileName(fnMovieOddMrc)
imgOutEven.setFileName(fnMovieEvenMrc)
imgOutOdd.setSamplingRate(movie.getSamplingRate())
imgOutEven.setSamplingRate(movie.getSamplingRate())
oddSet.append(imgOutOdd)
evenSet.append(imgOutEven)
oddSet.setSamplingRate(self.inputMovies.get().getSamplingRate())
evenSet.setSamplingRate(self.inputMovies.get().getSamplingRate())
self._defineOutputs(oddMovie=oddSet)
self._defineOutputs(evenMovie=evenSet)
self._defineSourceRelation(self.inputMovies, oddSet)
self._defineSourceRelation(self.inputMovies, evenSet)
if (self.sumFrames.get() is True):
oddSetAligned = self._createSetOfMicrographs(suffix='oddMic')
evenSetAligned = self._createSetOfMicrographs(suffix='evenMic')
for movie in self.inputMovies.get():
fnMovie = movie.getFileName()
fnMicOdd = self._getExtraPath(
pwutils.removeExt(basename(fnMovie)) + "_odd_aligned.mrc")
fnMicEven = self._getExtraPath(
pwutils.removeExt(basename(fnMovie)) + "_even_aligned.mrc")
imgOutOdd = Micrograph()
imgOutEven = Micrograph()
imgOutOdd.setFileName(fnMicOdd)
imgOutEven.setFileName(fnMicEven)
imgOutOdd.setSamplingRate(movie.getSamplingRate())
imgOutEven.setSamplingRate(movie.getSamplingRate())
oddSetAligned.append(imgOutOdd)
evenSetAligned.append(imgOutEven)
oddSetAligned.copyInfo(self.inputMovies.get())
evenSetAligned.copyInfo(self.inputMovies.get())
oddSetAligned.setSamplingRate(
self.inputMovies.get().getSamplingRate())
evenSetAligned.setSamplingRate(
self.inputMovies.get().getSamplingRate())
self._defineOutputs(oddMicrographs=oddSetAligned)
self._defineOutputs(evenMicrographs=evenSetAligned)
self._defineSourceRelation(self.inputMovies, oddSetAligned)
self._defineSourceRelation(self.inputMovies, evenSetAligned)
def _postprocessImageRow(self, img, imgRow):
# shortcut notation
prot = self.protocol
imgPath = self._imgPath
if self.ignoreIds:
img.setObjId(None) # Force to generate a new id in Set
if imgPath is not None:
if self._stackTrans is None:
self._stackTrans = FileTransform(
imgPath, prot._getExtraPath('Particles'), prot.copyFiles)
img.setFileName(self._stackTrans.transform(img.getFileName()))
if self._micIdOrName:
micId = imgRow.get('rlnMicrographId', None)
micName = imgRow.get('rlnMicrographName', None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = prot._getExtraPath('fake_micrograph%6d' % micId)
else:
if not len(self.micDict): # first time
if os.path.exists(os.path.join(imgPath, micName)):
micRoot = imgPath
else:
micRoot = pwutils.findRootFrom(
self._starFile, micName)
if micRoot is not None:
self._micTrans = FileTransform(
micRoot, prot._getExtraPath('Micrographs'),
prot.copyFiles)
if self._micTrans is not None:
micName = self._micTrans.transform(micName)
mic.setFileName(micName)
mic.setMicName(os.path.basename(micName))
mic.setAcquisition(img.getAcquisition())
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
img.setMicId(mic.getObjId())
if img.hasCoordinate():
coord = img.getCoordinate()
coord.setMicId(mic.getObjId())
coord.setMicName(os.path.basename(micName))
def micsFactory(index):
newMic = Micrograph(location=self.mic1)
return newMic
def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
# Load previously done items (from text file)
doneList = self._readDoneList()
# Check for newly done items
newDone = [
m.clone() for m in self.SetOfMicrographs
if int(m.getObjId()) not in doneList and self._isMicDone(m)
]
# We have finished when there is not more input micrographs (stream closed)
# and the number of processed micrographs is equal to the number of inputs
self.finished = self.streamClosed and (
len(doneList) + len(newDone)) == len(self.SetOfMicrographs)
streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN
if newDone:
self._writeDoneList(newDone)
elif not self.finished:
# If we are not finished and no new output have been produced
# it does not make sense to proceed and updated the outputs
# so we exit from the function here
return
outSet = self._loadOutputSet(SetOfMicrographs, 'micrographs.sqlite')
def tryToAppend(outSet, micOut, tries=1):
""" When micrograph is very big, sometimes it's not ready to be read
Then we will wait for it up to a minute in 6 time-growing tries. """
try:
outSet.append(micOut)
except Exception as ex:
micFn = micOut.getFileName() # Runs/..../extra/filename.mrc
errorStr = ('Image Extension: File %s has wrong size.' % micFn)
print(
"Output micrographs not ready, yet. Try: %d/6 (next in %fs)"
% (tries, tries * 3))
if errorStr in str(ex) and tries < 7:
from time import sleep
sleep(tries * 3)
tryToAppend(outSet, micOut, tries + 1)
else:
raise ex
for mic in newDone:
micOut = Micrograph()
if self.doDownsample:
micOut.setSamplingRate(
self.inputMicrographs.get().getSamplingRate() *
self.downFactor.get())
micOut.setObjId(mic.getObjId())
micOut.setFileName(self._getOutputMicrograph(mic))
micOut.setMicName(mic.getMicName())
tryToAppend(outSet, micOut)
self._updateOutputSet('outputMicrographs', outSet, streamMode)
if len(doneList) == 0: #firstTime
self._defineTransformRelation(self.inputMicrographs, outSet)
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(cons.STATUS_NEW)