def test_add(self):
""" Testing adding more groups or columns. """
og = OpticsGroups.create()
og1 = og.first()
self.assertEqual(len(og), 1)
self.assertAlmostEqual(og1.rlnVoltage, 300.)
og2 = og1._replace(rlnOpticsGroup=2,
rlnOpticsGroupName='opticsGroup2',
rlnVoltage=200.)
og.add(og2)
og3 = og1._replace(rlnOpticsGroup=3,
rlnOpticsGroupName='opticsGroup3',
rlnVoltage=100.)
og.add(og3)
self.assertEqual(len(og), 3)
og.addColumns(rlnMtfFileName='mtf_k2_200kV.star')
self.assertTrue(all(hasattr(ogx, 'rlnMtfFileName') for ogx in og))
og.update(2, rlnVoltage=300.)
og.update(3, rlnVoltage=300.)
for ogx in og:
self.assertAlmostEqual(ogx.rlnVoltage, 300.)
og.updateAll(rlnVoltage=200.)
for ogx in og:
self.assertAlmostEqual(ogx.rlnVoltage, 200.)
def createOutputStep(self):
imgSet = self.inputParticles.get()
outImgSet = self._createSetOfParticles()
outImgSet.copyInfo(imgSet)
outImgsFn = self._getFileName("output_star")
imgSet.setAlignmentProj()
if self.IS_GT30():
# self._optics = convert.getOpticsDict(outImgsFn)
mdIter = convert.Table.iterRows('particles@' + outImgsFn,
key='rlnImageId')
outImgSet.copyItems(imgSet,
updateItemCallback=self._updateItem31,
itemDataIterator=mdIter,
doClone=False)
og = OpticsGroups.fromStar(outImgsFn)
og.toImages(outImgSet)
else:
mdIter = md.iterRows(outImgsFn, sortByLabel=md.RLN_IMAGE_ID)
outImgSet.copyItems(imgSet,
updateItemCallback=self._updateItem30,
itemDataIterator=mdIter)
self._defineOutputs(outputParticles=outImgSet)
self._defineTransformRelation(self.inputParticles, outImgSet)
def _processMovie(self, movie):
movieFolder = self._getOutputMovieFolder(movie)
inputStar = os.path.join(movieFolder,
'%s_input.star' % self._getMovieRoot(movie))
pwutils.makePath(os.path.join(movieFolder, 'output'))
og = OpticsGroups.fromImages(self.inputMovies.get())
writer = convert.createWriter(optics=og)
# Let's use only the basename, since we will launch the command
# from the movieFolder
movie.setFileName(os.path.basename(movie.getFileName()))
writer.writeSetOfMovies([movie], inputStar)
# The program will run in the movie folder, so let's put
# the input files relative to that
args = "--i %s --o output/ " % os.path.basename(inputStar)
args += "--use_own "
f0, fN = self._getRange(movie)
args += "--first_frame_sum %d --last_frame_sum %d " % (f0, fN)
args += "--bin_factor %f --bfactor %d " % (self.binFactor,
self.bfactor)
args += "--angpix %0.5f " % (movie.getSamplingRate())
args += "--patch_x %d --patch_y %d " % (self.patchX, self.patchY)
args += "--group_frames %d " % self.groupFrames
args += "--j %d " % self.numberOfThreads
inputMovies = self.inputMovies.get()
if inputMovies.getGain():
args += ' --gainref "%s" ' % inputMovies.getGain()
args += ' --gain_rot %d ' % self.gainRot
args += ' --gain_flip %d ' % self.gainFlip
if self.IS_GT30():
if self.defectFile.get():
args += ' --defect_file "%s" ' % self.defectFile.get()
if self._savePsSum():
args += ' --grouping_for_ps %d ' % self._calcPsDose()
if self.doDW:
args += "--dose_weighting "
preExp, dose = self._getCorrectedDose(self.inputMovies.get())
args += "--dose_per_frame %f " % dose
args += "--preexposure %f " % preExp
if self.saveNonDW:
args += " --save_noDW "
if self.extraParams.hasValue():
args += " " + self.extraParams.get()
try:
self.runJob(self._getProgram(), args, cwd=movieFolder)
self._computeExtra(movie)
self._moveFiles(movie)
except:
print("ERROR: processing movie: ", movie.getFileName())
def _createSetOfMics(self, n=10, nOptics=2):
micName = 'BPV_13%02d.mrc'
psdName = 'BPV_13%02d_PSD.ctf:mrc'
ogName = 'opticsGroup%d'
mtfFile = 'mtfFile%d.star'
cleanPath(self.getOutputPath('micrographs.sqlite'))
micsDb = self.getOutputPath('micrographs.sqlite')
outputMics = SetOfMicrographs(filename=micsDb)
outputMics.setSamplingRate(1.234)
mic = SetOfMicrographs.ITEM_TYPE()
acq = Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000)
og = OpticsGroups.create(rlnMtfFileName='')
fog = og.first()
ctf = CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15)
outputMics.setAcquisition(acq)
mic.setAcquisition(acq)
mic.setCTF(ctf)
itemsPerOptics = n // nOptics
for i in range(1, n + 1):
mic.setFileName(micName % i)
ctf = mic.getCTF()
ctf.setPsdFile(psdName % i)
ctf.setFitQuality(np.random.uniform())
ctf.setResolution(np.random.uniform(3, 15))
ogNumber = (i - 1) // itemsPerOptics + 1
ogDict = {
'rlnOpticsGroup': ogNumber,
'rlnOpticsGroupName': ogName % ogNumber,
'rlnMtfFileName': mtfFile % ogNumber
}
if ogNumber in og:
og.update(ogNumber, **ogDict)
else:
og.add(fog._replace(**ogDict))
mic.rlnOpticsGroup = ogNumber
mic.setObjId(None)
outputMics.append(mic)
print(">>> Writing micrograph set to: ", micsDb)
outputMics.write()
return outputMics
def test_readSetOfParticlesAfterCtf(self):
if not Plugin.IS_GT30():
print("Skipping test (required Relion > 3.1)")
return
starFile = self.ds.getFile(
"CtfRefine/job023/particles_ctf_refine.star")
partsReader = Table.Reader(starFile, tableName='particles')
firstRow = partsReader.getRow()
partsSet = self.__readParticles(starFile)
first = partsSet.getFirstItem()
ogLabels = ['rlnBeamTiltX', 'rlnBeamTiltY']
extraLabels = ['rlnCtfBfactor', 'rlnCtfScalefactor', 'rlnPhaseShift']
for l in extraLabels:
value = getattr(first, '_%s' % l)
self.assertIsNotNone(value, "Missing label: %s" % l)
self.assertAlmostEqual(getattr(firstRow, l), value)
fog = OpticsGroups.fromImages(partsSet).first()
self.assertTrue(all(hasattr(fog, l) for l in ogLabels))
# Also test writing and preserving extra labels
outputStar = self.getOutputPath('particles.star')
print(">>> Writing to particles star: %s" % outputStar)
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partsSet, outputStar)
fog = OpticsGroups.fromStar(outputStar).first()
self.assertTrue(all(hasattr(fog, l) for l in ogLabels))
partsReader = Table.Reader(outputStar, tableName='particles')
firstRow = partsReader.getRow()
for l in extraLabels:
value = getattr(first, '_%s' % l)
self.assertIsNotNone(value, "Missing label: %s" % l)
self.assertAlmostEqual(getattr(firstRow, l), value)
def _updateOutputSet(self, outputName, outputSet,
state=Set.STREAM_OPEN):
""" Redefine this method to update optics info. """
first = getattr(self, '_firstUpdate', True)
if first:
og = OpticsGroups.fromImages(outputSet)
og.updateAll(rlnImagePixelSize=self._getNewSampling(),
rlnImageSize=self.getNewImgSize())
og.toImages(outputSet)
ProtExtractParticles._updateOutputSet(self, outputName, outputSet,
state=state)
self._firstUpdate = False
def test_fromStar(self):
mtfStar = self.getOutputPath('input_mtf.star')
f = open(mtfStar, 'w')
f.write("""
data_optics
loop_
_rlnOpticsGroupName #1
_rlnOpticsGroup #2
_rlnMtfFileName #3
_rlnMicrographOriginalPixelSize #4
_rlnVoltage #5
_rlnSphericalAberration #6
_rlnAmplitudeContrast #7
_rlnMicrographPixelSize #8
opticsGroup1 1 mtf_k2_200kV.star 0.885000 200.000000 1.400000 0.100000 0.885000
opticsGroup2 2 mtf_k2_200kV.star 0.885000 200.000000 1.400000 0.100000 0.885000
opticsGroup3 3 mtf_k2_200kV.star 0.885000 200.000000 1.400000 0.100000 0.885000
data_micrographs
loop_
data_micrographs
loop_
_rlnMicrographName #1
_rlnOpticsGroup #2
20170629_00025_frameImage.tiff 1
20170629_00035_frameImage.tiff 2
20170629_00045_frameImage.tiff 3
""")
f.close()
print(magentaStr("\nRunning relion - assign optics groups:"))
protAssign = self.newProtocol(ProtRelionAssignOpticsGroup,
objLabel='assign optics - from star',
inputSet=self.protImport.outputMovies,
inputType=1, # from star file
inputStar=mtfStar)
protAssign = self.launchProtocol(protAssign)
outputMovies = protAssign.outputMovies
og = OpticsGroups.fromImages(outputMovies)
self.assertEqual(3, len(og))
for i, movie in enumerate(outputMovies):
self.assertEqual(i + 1, movie.getAttributeValue('_rlnOpticsGroup'))
def _updateOutputSet(self,
outputName,
outputSet,
state=pwobj.Set.STREAM_OPEN):
""" Redefine this method to update optics info. """
if outputName not in self.updatedSets:
og = OpticsGroups.fromImages(outputSet)
og.updateAll(rlnMicrographOriginalPixelSize=self.inputMovies.get().
getSamplingRate())
og.toImages(outputSet)
self.updatedSets.append(outputName)
ProtAlignMovies._updateOutputSet(self,
outputName,
outputSet,
state=state)
def test_fromRefine3D(self):
""" Import particles from Refine3D job star file.
"""
if not relion.Plugin.IS_GT30():
return
starFile = self.ds.getFile('Refine3D/job019/run_it020_data.star')
optics = OpticsGroups.fromStar(starFile).first()
prot1 = self.newProtocol(emprot.ProtImportParticles,
objLabel='from relion (refine 3d)',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_RELION,
starFile=starFile,
magnification=10000,
samplingRate=optics.rlnImagePixelSize,
haveDataBeenPhaseFlipped=False
)
self.launchProtocol(prot1)
self.checkOutput(prot1, 'outputParticles', ['outputParticles.hasAlignmentProj()'])
def test_fromExtract(self):
""" Import particles.star from Extract job.
"""
if not relion.Plugin.IS_GT30():
return
starFile = self.ds.getFile('Extract/job018/particles.star')
optics = OpticsGroups.fromStar(starFile).first()
prot1 = self.newProtocol(emprot.ProtImportParticles,
objLabel='from relion (extract job)',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_RELION,
starFile=starFile,
magnification=10000,
samplingRate=optics.rlnImagePixelSize,
haveDataBeenPhaseFlipped=False
)
self.launchProtocol(prot1)
self.checkOutput(prot1, 'outputParticles', [])
def test_fromStar(self):
if not Plugin.IS_GT30():
print("Skipping test (required Relion > 3.1)")
return
partsStar = self.ds.getFile("Extract/job018/particles.star")
print("<<< Reading optics groups from file: \n %s\n" % partsStar)
og = OpticsGroups.fromStar(partsStar)
fog = og.first()
# test hasColumn method
for colName in ['rlnMtfFileName', 'rlnOpticsGroupName']:
self.assertTrue(og.hasColumn(colName))
# acq = first.getAcquisition()
self.assertEqual(fog.rlnMtfFileName, 'mtf_k2_200kV.star')
self.assertEqual(fog.rlnOpticsGroupName, 'opticsGroup1')
self.assertEqual(og['opticsGroup1'], fog)
def _updateOutputSet(self,
outputName,
outputSet,
state=pwobj.Set.STREAM_OPEN):
""" Redefine this method to set EER attrs. """
first = getattr(self, '_firstUpdate', True)
if first and outputName == 'outputMovies':
og = OpticsGroups.fromImages(outputSet)
if self.isEER:
og.updateAll(rlnEERGrouping=self.eerGroup.get(),
rlnEERUpsampling=self.eerSampling.get() + 1)
og.updateAll(rlnMicrographStartFrame=self.alignFrame0.get())
og.toImages(outputSet)
ProtAlignMovies._updateOutputSet(self,
outputName,
outputSet,
state=state)
self._firstUpdate = False
def test_readSetOfParticles(self):
if not Plugin.IS_GT30():
print("Skipping test (required Relion > 3.1)")
return
partsSet = self.__readParticles(
self.ds.getFile("Extract/job018/particles.star"),
extraLabels=['rlnNrOfSignificantSamples'])
partsSet.write()
first = partsSet.getFirstItem()
first.printAll()
self.assertAlmostEqual(first.getSamplingRate(), 1.244531)
self.assertEqual(first.getClassId(), 4)
self.assertTrue(hasattr(first, '_rlnNrOfSignificantSamples'))
fog = OpticsGroups.fromImages(partsSet).first()
self.assertEqual(fog.rlnMtfFileName, 'mtf_k2_200kV.star')
self.assertEqual(fog.rlnOpticsGroupName, 'opticsGroup1')
def _extractMicrographList(self, micList, params):
workingDir = self.getWorkingDir()
micsStar = self._getMicsStar(micList)
og = OpticsGroups.fromImages(self.getInputMicrographs())
starWriter = relion.convert.createWriter(rootDir=workingDir,
outputDir=self._getTmpPath(),
optics=og)
starWriter.writeSetOfMicrographs(micList, self._getPath(micsStar))
# convert.writeSetOfMicrographs(micList, self._getPath(micsStar),
# alignType=emcts.ALIGN_NONE,
# preprocessImageRow=self._preprocessMicrographRow)
partsStar = self._getMicParticlesStar(micList)
for mic in micList:
self._convertCoordinates(mic, self.coordDict[mic.getObjId()])
self._micCoordStarDict[mic.getObjId()] = partsStar
args = ' --i %s --part_star %s %s' % (micsStar, partsStar, params)
self.runJob(self._getProgram('relion_preprocess'), args, cwd=workingDir)
def test_string(self):
if not Plugin.IS_GT30():
print("Skipping test (required Relion > 3.1)")
return
og = OpticsGroups.create(rlnMtfFileName='mtf_k2_200kV.star')
fog = og.first()
# acq = first.getAcquisition()
self.assertEqual(fog.rlnMtfFileName, 'mtf_k2_200kV.star')
self.assertEqual(fog.rlnOpticsGroupName, 'opticsGroup1')
self.assertEqual(og['opticsGroup1'], fog)
# try update by id
og.update(1, rlnMtfFileName="new_mtf_k2.star")
# try update by name
og.update('opticsGroup1', rlnImageSize=512)
fog = og.first()
# acq = first.getAcquisition()
self.assertEqual(fog.rlnMtfFileName, 'new_mtf_k2.star')
self.assertEqual(fog.rlnImageSize, 512)
self.assertEqual(fog.rlnOpticsGroupName, 'opticsGroup1')
self.assertEqual(og['opticsGroup1'], fog)
def _checkAcq(output):
og = OpticsGroups.fromImages(output)
fog = og.first()
self.assertEqual(fog.rlnOpticsGroupName, self.GROUP_NAME)
self.assertEqual(os.path.basename(fog.rlnMtfFileName),
os.path.basename(self.MTF_FILE))
def createOutputStep(self, inputId):
inputSet = self.inputSet.get()
getMicName = lambda item: item.getMicName()
if isinstance(inputSet, SetOfMovies):
outputSet = self._createSetOfMovies()
outputName = 'outputMovies'
elif isinstance(inputSet, SetOfMicrographs):
outputSet = self._createSetOfMicrographs()
outputName = 'outputMicrographs'
elif isinstance(inputSet, SetOfParticles):
outputSet = self._createSetOfParticles()
outputName = 'outputParticles'
getMicName = lambda item: item.getCoordinate().getMicName()
else:
raise Exception("Invalid input of type %s, expecting:\n"
"SetOfMovies, SetOfMicrographs or SetOfParticles"
% inputSet.getClassName())
# Copy general info from input set
outputSet.copyInfo(inputSet)
if self.inputType == 0: # single group params
og = OpticsGroups.fromImages(inputSet)
outputSet.copyItems(inputSet)
if len(og) > 1:
raise Exception("Multiple optics groups detected in the input!\n"
"Assigning single optics group params "
"is valid only when the input set contains "
"one optics group.")
acq = inputSet.getAcquisition()
params = {
'rlnVoltage': acq.getVoltage(),
'rlnSphericalAberration': acq.getSphericalAberration(),
'rlnAmplitudeContrast': acq.getAmplitudeContrast(),
getPixelSizeLabel(inputSet): inputSet.getSamplingRate(),
'rlnImageSize': inputSet.getXDim(),
'rlnOpticsGroupName': self.opticsGroupName.get(),
'rlnBeamTiltX': self.beamTiltX.get(),
'rlnBeamTiltY': self.beamTiltY.get()
}
og = OpticsGroups.create(**params)
if self.mtfFile.hasValue():
inputMtf = self.mtfFile.get()
outputMtf = self._getPath(os.path.basename(inputMtf))
pwutils.copyFile(inputMtf, outputMtf)
og.addColumns(rlnMtfFileName=outputMtf)
if self.gainFile.hasValue():
gainFn = self._convertGain()
og.addColumns(rlnMicrographGainName=gainFn)
if self.defectFile.hasValue():
inputDef = self.defectFile.get()
outputDef = self._getPath(os.path.basename(inputDef))
pwutils.copyFile(inputDef, outputDef)
og.addColumns(rlnMicrographDefectFile=outputDef)
else:
inputStar = self.inputStar.get()
og = OpticsGroups.fromStar(inputStar)
micTable = emtable.Table(fileName=inputStar,
tableName='micrographs')
micDict = {row.rlnMicrographName: row.rlnOpticsGroup
for row in micTable}
# check if MTF file exists
if og.hasColumn('rlnMtfFileName'):
for i in og:
if not os.path.exists(i.rlnMtfFileName):
self.warning("MTF file %s not found for %s" % (
i.rlnMtfFileName, i.rlnOpticsGroupName
))
# check if gain file exists
if og.hasColumn('rlnMicrographGainName'):
for i in og:
if not pwutils.exists(i.rlnMicrographGainName):
self.warning("Gain reference file %s not found for %s" % (
i.rlnMicrographGainName, i.rlnOpticsGroupName
))
def updateItem(item, row):
micName = getMicName(item)
if micName not in micDict:
raise Exception("Micrograph name (aka micName) '%s' was "
"not found in the 'data_micrographs' table of "
"the input star file: %s"
% (micName, inputStar))
ogNumber = micDict[micName]
if not hasattr(item, '_rlnOpticsGroup'):
item._rlnOpticsGroup = Integer()
item._rlnOpticsGroup.set(ogNumber)
outputSet.copyItems(inputSet,
updateItemCallback=updateItem,
doClone=False)
print(og)
og.toImages(outputSet)
self._defineOutputs(**{outputName: outputSet})
self._defineTransformRelation(inputSet, outputSet)
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
given the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print("\n")
print("*" * 80)
print("* Launching test: ", fileKey)
print("*" * 80)
is2D = alignType == ALIGN_2D
if fileKey == 'alignShiftRotExp':
# relion requires mrcs stacks
origFn = self.dataset.getFile(fileKey)
stackFn = replaceExt(origFn, ".mrcs")
createLink(origFn, stackFn)
else:
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile("reconstruction/gold/" + fileKey +
'_Gold_rln_output.vol')
if PRINT_FILES:
print("BINARY DATA: ", stackFn)
print("SET1: ", partFn1)
print(" MD: ", mdFn)
print("SET2: ", partFn2)
print("OUTPUT: ", outputFn)
print("GOLD: ", goldFn)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
acq = Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000)
og = OpticsGroups.create(rlnMtfFileName="mtfFile1.star",
rlnImageSize=128)
partSet.setSamplingRate(1.0)
partSet.setAcquisition(acq)
og.toImages(partSet)
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [np.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print("Partset", partFn1)
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partSet, mdFn, alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
convert.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
convert.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
convert.readSetOfParticles(mdFn,
partSet2,
rowToFunc=convert.rowToVolume,
alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print("-" * 5)
print(img.getFileName(), img.getIndex())
print('m1:\n', m1, convert.geometryFromMatrix(m1, False))
print('m2:\n', m2, convert.geometryFromMatrix(m2, False))
self.assertTrue(np.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
if CLEAN_IMAGES:
cleanPath(outputFn)