def _checkNewMics(self, micSet):
""" Check for already computed CTF and update the output set. """
micDict = {}
newMic = False
for mic in micSet:
micDict[mic.getFileName()] = True
if micDict:
if micSet.getSize():
micSet.enableAppend()
micSet.loadAllProperties()
else:
micSet.setStreamState(micSet.STREAM_OPEN)
acquisition = Acquisition()
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
micSet.setAcquisition(acquisition)
micSet.setSamplingRate(self.samplingRate.get())
mic = Micrograph()
counter = 0
for k, v in self.xmippMic.iteritems():
counter += 1
if (k not in micDict):
mic.setFileName(k)
mic.setMicName(basename(k))
mic.setObjId(counter)
micSet.append(mic)
newMic = True
return micSet, newMic #why a dictionary, a boolean may be enought
def createSetOfParticles(self, setPartSqliteName, partFn,
doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
#defocus=15000 + 5000* random.random()
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i]#+500.
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(Transform(a))
self.partSet.append(p)
self.partSet.write()
def _particleFromEmx(emxObj, particle):
_imageFromEmx(emxObj, particle)
_setCoordinatesFromEmx(emxObj, particle.getCoordinate())
mic = emxObj.getMicrograph()
if mic is not None:
acquisition = Acquisition()
_acquisitionFromEmx(mic, acquisition)
particle.setAcquisition(acquisition)
particle.hasAcquisition()
def _checkNewItems(self, objSet):
""" Check for already computed micrograph/movie and
update the output set. """
objDict = {}
newObj = False
for obj in objSet:
objDict[obj.getFileName()] = True
if objDict:
if objSet.getSize():
objSet.enableAppend()
objSet.loadAllProperties()
else:
objSet.setStreamState(objSet.STREAM_OPEN)
acquisition = Acquisition()
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(self.samplingRate.get())
if self.setof == SET_OF_MOVIES:
obj = Movie()
elif self.setof == SET_OF_MICROGRAPHS:
obj = Micrograph()
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
obj = Micrograph()
else:
raise Exception('Unknown data type')
counter = 0
for k, v in self.dictObj.iteritems():
counter += 1
if (k not in objDict):
obj.setFileName(k)
obj.setMicName(basename(k))
obj.setObjId(counter)
objSet.append(obj)
newObj = True
return objSet, newObj # why a dictionary, a boolean may be enought
def _checkNewItems(self, objSet):
""" Check for already computed micrograph/movie and
update the output set. """
objDict = {}
newObj = False
for obj in objSet:
objDict[obj.getFileName()] = True
if objDict:
if objSet.getSize():
objSet.enableAppend()
objSet.loadAllProperties()
else:
objSet.setStreamState(objSet.STREAM_OPEN)
acquisition = Acquisition()
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(self.samplingRate.get())
if self.setof == SET_OF_MOVIES:
obj = Movie()
elif self.setof == SET_OF_MICROGRAPHS:
obj = Micrograph()
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
obj = Micrograph()
else:
raise Exception('Unknown data type')
counter = 0
for k, v in self.dictObj.iteritems():
counter += 1
if (k not in objDict):
obj.setFileName(k)
obj.setMicName(basename(k))
obj.setObjId(counter)
objSet.append(obj)
newObj = True
return objSet, newObj # why a dictionary, a boolean may be enought
def aaatest_particlesToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles.star')
fnStk = self.getOutputPath('particles.stk')
print ">>> Writing to file: %s" % fnStar
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def getAcquisition(self):
""" Build and fill an acquisition object. """
acquisition = Acquisition()
self.fillAcquisition(acquisition)
return acquisition
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i % 3)
img.setClassId(i % 5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(
ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception(
'Import of images: %s, failed. outputParticles is None.' %
inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [
protSplitSet.outputParticles01, protSplitSet.outputParticles02
]
outputSet = SetOfParticles(
filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def processMovieStep(self, movieDict, hasAlignment):
movie = Movie()
movie.setAcquisition(Acquisition())
if hasAlignment:
movie.setAlignment(MovieAlignment())
movie.setAttributesFromDict(movieDict,
setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
if (self.isContinued() and os.path.exists(self._getMovieDone(movie))):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(self._getMovieDone(movie))
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2',
'-d -f %s' % movieName,
cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.tif'):
#FIXME: It seems that we have some flip problem with compressed
# tif files, we need to check that
newMovieName = movieName.replace('.tif', '.mrc')
# we assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tif2mrc',
'%s %s' % (movieName, newMovieName),
cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin,
line.strip())
ih.convert(
inputFrame,
(i + 1, os.path.join(movieFolder,
newMovieName)))
else:
newMovieName = movieName
if (self.CONVERT_TO_MRC
and not (newMovieName.endswith("mrc")
or newMovieName.endswith("mrcs"))):
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
newMovieName = pwutils.replaceExt(newMovieName,
self.CONVERT_TO_MRC)
outputMovieFn = os.path.join(movieFolder, newMovieName)
self.info("Converting movie '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
ImageHandler().convertStack(inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if pwutils.envVarOn('SCIPION_DEBUG_NOCLEAN'):
self.info('Clean movie data DISABLED. '
'Movie folder will remain in disk!!!')
else:
self.info("Erasing.....movieFolder: %s" % movieFolder)
os.system('rm -rf %s' % movieFolder)
# cleanPath(movieFolder)
# Mark this movie as finished
open(self._getMovieDone(movie), 'w').close()
def processMovieStep(self, movieDict, hasAlignment):
movie = Movie()
movie.setAcquisition(Acquisition())
if hasAlignment:
movie.setAlignment(MovieAlignment())
movie.setAttributesFromDict(movieDict, setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
movieDoneFn = self._getMovieDone(movie)
if (self.isContinued() and os.path.exists(movieDoneFn)):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(movieDoneFn)
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2', '-d -f %s' % movieName, cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin, line.strip())
ih.convert(inputFrame,
(i+1, os.path.join(movieFolder, newMovieName)))
else:
newMovieName = movieName
convertExt = self._getConvertExtension(newMovieName)
correctGain = self._doCorrectGain()
if convertExt or correctGain:
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
if convertExt:
newMovieName = pwutils.replaceExt(newMovieName, convertExt)
else:
newMovieName = '%s_corrected.%s' % os.path.splitext(newMovieName)
outputMovieFn = os.path.join(movieFolder, newMovieName)
# If the protocols wants Scipion to apply the gain, then
# there is no reason to convert, since we can produce the
# output in the format expected by the program. In some cases,
# the alignment programs can directly deal with gain and dark
# correction images, so we don't need to apply it
if self._doCorrectGain():
self.info("Correcting gain and dark '%s' -> '%s'"
% (inputMovieFn, outputMovieFn))
gain, dark = self.getGainAndDark()
self.correctGain(inputMovieFn, outputMovieFn,
gainFn=gain, darkFn=dark)
else:
self.info("Converting movie '%s' -> '%s'"
% (inputMovieFn, outputMovieFn))
ImageHandler().convertStack(inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if self._doMovieFolderCleanUp():
self._cleanMovieFolder(movieFolder)
# Mark this movie as finished
open(movieDoneFn, 'w').close()
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
give the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print "\n"
print "*" * 80
print "* Launching test: ", fileKey
print "*" * 80
is2D = alignType == ALIGN_2D
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')
if PRINT_FILES:
print "BINARY DATA: ", stackFn
print "SET1: ", partFn1
print " MD: ", mdFn
print "SET2: ", partFn2
print "OUTPUT: ", outputFn
print "GOLD: ", goldFn
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
partSet.setAcquisition(
Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [numpy.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print "Parset", partFn1
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.writeSetOfParticles(partSet,
mdFn,
"/tmp",
alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print "-" * 5
print img.getFileName(), img.getIndex()
print 'm1:\n', m1, relion.geometryFromMatrix(m1, False)
print 'm2:\n', m2, relion.geometryFromMatrix(m2, False)
# self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i%3)
img.setClassId(i%5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5
)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02]
outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )