def testRead(self):
""" Read micrographs from a an sqlite file.
It should contains Acquisition info. """
micFn = self.dataset.getFile('micsGoldSqlite2')
print(">>> Reading gold micrographs from: ", micFn)
micSet = emobj.SetOfMicrographs(filename=micFn)
self.assertEqual(2, micSet.getSize())
acquisition = emobj.Acquisition()
acquisition.setMagnification(10000.)
acquisition.setVoltage(200.)
acquisition.setSphericalAberration(2.26)
acquisition.setAmplitudeContrast(0.1)
mic2 = emobj.Micrograph()
mic2.setSamplingRate(2.8)
mic2.setAcquisition(acquisition)
fileNames = [
'/home/roberto/Scipion/Test/Test2/down2_12585',
'/home/roberto/Scipion/Test/Test2/down2_12610'
]
counter = 0
for mic in micSet:
mic2.setFileName(fileNames[counter])
self.assertTrue(mic.equalAttributes(mic2))
counter += 1
def rowToAcquisition(acquisitionRow):
""" Create an acquisition from a row of a meta """
if acquisitionRow.containsAll(ACQUISITION_DICT):
acquisition = pwobj.Acquisition()
rowToObject(acquisitionRow, acquisition, ACQUISITION_DICT)
else:
acquisition = None
return acquisition
def setUpClass(cls):
setupTestOutput(cls)
cls.dataset = DataSet.getDataSet('xmipp_tutorial')
cls.dbGold = cls.dataset.getFile('micsGoldSqlite')
cls.micsPattern = cls.dataset.getFile('allMics')
cls.dbFn = cls.getOutputPath('micrographs.sqlite')
cls.acquisition = emobj.Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1)
# cls.mics = glob(cls.micsPattern)
cls.mics = []
for mic in iglob(cls.micsPattern):
cls.mics.append(cls.getRelPath(cls.dataset.getPath(), mic))
if len(cls.mics) == 0:
raise Exception('There are not micrographs matching pattern')
cls.mics.sort()
def processMovieStep(self, movieDict, hasAlignment):
movie = emobj.Movie()
movie.setAcquisition(emobj.Acquisition())
if hasAlignment:
movie.setAlignment(emobj.MovieAlignment())
movie.setAttributesFromDict(movieDict,
setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
movieDoneFn = self._getMovieDone(movie)
if self.isContinued() and os.path.exists(movieDoneFn):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(movieDoneFn)
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2',
'-d -f %s' % movieName,
cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = emlib.image.ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin,
line.strip())
ih.convert(
inputFrame,
(i + 1, os.path.join(movieFolder,
newMovieName)))
else:
newMovieName = movieName
convertExt = self._getConvertExtension(newMovieName)
correctGain = self._doCorrectGain()
if convertExt or correctGain:
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
if convertExt:
newMovieName = pwutils.replaceExt(newMovieName, convertExt)
else:
newMovieName = '%s_corrected.%s' % os.path.splitext(
newMovieName)
outputMovieFn = os.path.join(movieFolder, newMovieName)
# If the protocols wants Scipion to apply the gain, then
# there is no reason to convert, since we can produce the
# output in the format expected by the program. In some cases,
# the alignment programs can directly deal with gain and dark
# correction images, so we don't need to apply it
if self._doCorrectGain():
self.info("Correcting gain and dark '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
gain, dark = self.getGainAndDark()
self.correctGain(inputMovieFn,
outputMovieFn,
gainFn=gain,
darkFn=dark)
else:
self.info("Converting movie '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
emlib.image.ImageHandler().convertStack(
inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if self._doMovieFolderCleanUp():
self._cleanMovieFolder(movieFolder)
# Mark this movie as finished
open(movieDoneFn, 'w').close()
def _checkNewItems(self, objSet):
""" Check for already computed micrograph/movie and
update the output set. """
objDict = {}
newObj = False
for obj in objSet:
objDict[obj.getFileName()] = True
if objDict:
if objSet.getSize():
objSet.enableAppend()
objSet.loadAllProperties()
else:
objSet.setStreamState(objSet.STREAM_OPEN)
acquisition = emobj.Acquisition()
if self.setof == SET_OF_MICROGRAPHS:
acquisition.setMagnification(
self.inputMics.get().getAcquisition().getMagnification())
acquisition.setVoltage(
self.inputMics.get().getAcquisition().getVoltage())
acquisition.setSphericalAberration(
self.inputMics.get().getAcquisition().getSphericalAberration())
acquisition.setAmplitudeContrast(
self.inputMics.get().getAcquisition().getAmplitudeContrast())
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(self.inputMics.get().getSamplingRate())
elif self.setof == SET_OF_MOVIES:
acquisition.setMagnification(
self.inputMovies.get().getAcquisition().getMagnification())
acquisition.setVoltage(
self.inputMovies.get().getAcquisition().getVoltage())
acquisition.setSphericalAberration(
self.inputMovies.get().getAcquisition().getSphericalAberration())
acquisition.setAmplitudeContrast(
self.inputMovies.get().getAcquisition().getAmplitudeContrast())
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(
self.inputMovies.get().getSamplingRate())
else:
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
objSet.setAcquisition(acquisition)
if self.setof == SET_OF_PARTICLES:
objSet.setSamplingRate(
self.inputParticles.get().getSamplingRate())
else:
objSet.setSamplingRate(self.samplingRate.get())
if self.setof == SET_OF_MOVIES:
obj = emobj.Movie()
elif self.setof == SET_OF_MICROGRAPHS:
obj = emobj.Micrograph()
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
obj = emobj.Micrograph()
elif self.setof == SET_OF_PARTICLES:
obj = emobj.Particle()
else:
raise Exception('Unknown data type')
for k, v in self.dictObj.items():
if k not in objDict:
self.counter += 1
obj.setFileName(k)
if self.setof != SET_OF_PARTICLES:
obj.setMicName(basename(k))
obj.setObjId(self.counter)
objSet.append(obj)
newObj = True
return objSet, newObj # why a dictionary, a boolean may be enough
