def test_particlesToStar(self):
""" Write a SetOfParticles to Relion star input file. """
outputStar = self.getOutputPath("particles.star")
outputParts = self._createSetOfParts(10, 2, 10)
print(">>> Writing to particles star: %s" % outputStar)
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(outputParts, outputStar)
def test_micrographsToStar(self):
""" Write a SetOfParticles to Relion star input file. """
outputStar = self.getOutputPath("micrographs.star")
outputMics = self._createSetOfMics(10)
print(">>> Writing to micrographs: %s" % outputStar)
starWriter = convert.createWriter()
starWriter.writeSetOfMicrographs(outputMics, outputStar)
def test_particlesImportToStar(self):
sqliteFn = self.ds.getFile("import/case2/particles.sqlite")
partsSet = SetOfParticles(filename=sqliteFn)
partsSet.loadAllProperties()
outputStar = self.getOutputPath("particles.star")
print(">>> Writing to particles star: %s" % outputStar)
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partsSet, outputStar)
def _processMovie(self, movie):
movieFolder = self._getOutputMovieFolder(movie)
inputStar = os.path.join(movieFolder,
'%s_input.star' % self._getMovieRoot(movie))
pwutils.makePath(os.path.join(movieFolder, 'output'))
og = OpticsGroups.fromImages(self.inputMovies.get())
writer = convert.createWriter(optics=og)
# Let's use only the basename, since we will launch the command
# from the movieFolder
movie.setFileName(os.path.basename(movie.getFileName()))
writer.writeSetOfMovies([movie], inputStar)
# The program will run in the movie folder, so let's put
# the input files relative to that
args = "--i %s --o output/ " % os.path.basename(inputStar)
args += "--use_own "
f0, fN = self._getRange(movie)
args += "--first_frame_sum %d --last_frame_sum %d " % (f0, fN)
args += "--bin_factor %f --bfactor %d " % (self.binFactor,
self.bfactor)
args += "--angpix %0.5f " % (movie.getSamplingRate())
args += "--patch_x %d --patch_y %d " % (self.patchX, self.patchY)
args += "--group_frames %d " % self.groupFrames
args += "--j %d " % self.numberOfThreads
inputMovies = self.inputMovies.get()
if inputMovies.getGain():
args += ' --gainref "%s" ' % inputMovies.getGain()
args += ' --gain_rot %d ' % self.gainRot
args += ' --gain_flip %d ' % self.gainFlip
if self.IS_GT30():
if self.defectFile.get():
args += ' --defect_file "%s" ' % self.defectFile.get()
if self._savePsSum():
args += ' --grouping_for_ps %d ' % self._calcPsDose()
if self.doDW:
args += "--dose_weighting "
preExp, dose = self._getCorrectedDose(self.inputMovies.get())
args += "--dose_per_frame %f " % dose
args += "--preexposure %f " % preExp
if self.saveNonDW:
args += " --save_noDW "
if self.extraParams.hasValue():
args += " " + self.extraParams.get()
try:
self.runJob(self._getProgram(), args, cwd=movieFolder)
self._computeExtra(movie)
self._moveFiles(movie)
except:
print("ERROR: processing movie: ", movie.getFileName())
def _pickMicrographList(self, micList, *args):
if not micList:
return
micsDir = self._createTmpMicsDir(micList)
micStar = os.path.join(micsDir, 'input_micrographs.star')
writer = convert.createWriter(rootDir=micsDir, outputDir=micsDir)
writer.writeSetOfMicrographs(micList, micStar)
self._pickMicrographsFromStar(micStar, micsDir, *args)
# Move coordinates files to tmp
os.system('mv %s/*autopick.star %s/' % (micsDir, self._getTmpPath()))
def convertInputStep(self, moviesId, subset):
self.info("Relion version:")
self.runJob("relion_convert_to_tiff --version", "", numberOfMpi=1)
self.info("Detected version from config: %s"
% relion.Plugin.getActiveVersion())
moviesList = self.inputMovies.get()
if subset > 0:
moviesList = [m.clone()
for i, m in enumerate(moviesList) if i < subset]
micStar = self._getTmpPath('input_movies.star')
tmp = self._getTmpPath()
writer = convert.createWriter(rootDir=tmp, outputDir=tmp)
writer.writeSetOfMovies(moviesList, micStar)
def test_readSetOfParticlesAfterCtf(self):
if not Plugin.IS_GT30():
print("Skipping test (required Relion > 3.1)")
return
starFile = self.ds.getFile(
"CtfRefine/job023/particles_ctf_refine.star")
partsReader = Table.Reader(starFile, tableName='particles')
firstRow = partsReader.getRow()
partsSet = self.__readParticles(starFile)
first = partsSet.getFirstItem()
ogLabels = ['rlnBeamTiltX', 'rlnBeamTiltY']
extraLabels = ['rlnCtfBfactor', 'rlnCtfScalefactor', 'rlnPhaseShift']
for l in extraLabels:
value = getattr(first, '_%s' % l)
self.assertIsNotNone(value, "Missing label: %s" % l)
self.assertAlmostEqual(getattr(firstRow, l), value)
fog = OpticsGroups.fromImages(partsSet).first()
self.assertTrue(all(hasattr(fog, l) for l in ogLabels))
# Also test writing and preserving extra labels
outputStar = self.getOutputPath('particles.star')
print(">>> Writing to particles star: %s" % outputStar)
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partsSet, outputStar)
fog = OpticsGroups.fromStar(outputStar).first()
self.assertTrue(all(hasattr(fog, l) for l in ogLabels))
partsReader = Table.Reader(outputStar, tableName='particles')
firstRow = partsReader.getRow()
for l in extraLabels:
value = getattr(first, '_%s' % l)
self.assertIsNotNone(value, "Missing label: %s" % l)
self.assertAlmostEqual(getattr(firstRow, l), value)
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
given the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print("\n")
print("*" * 80)
print("* Launching test: ", fileKey)
print("*" * 80)
is2D = alignType == ALIGN_2D
if fileKey == 'alignShiftRotExp':
# relion requires mrcs stacks
origFn = self.dataset.getFile(fileKey)
stackFn = replaceExt(origFn, ".mrcs")
createLink(origFn, stackFn)
else:
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile("reconstruction/gold/" + fileKey +
'_Gold_rln_output.vol')
if PRINT_FILES:
print("BINARY DATA: ", stackFn)
print("SET1: ", partFn1)
print(" MD: ", mdFn)
print("SET2: ", partFn2)
print("OUTPUT: ", outputFn)
print("GOLD: ", goldFn)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
acq = Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000)
og = OpticsGroups.create(rlnMtfFileName="mtfFile1.star",
rlnImageSize=128)
partSet.setSamplingRate(1.0)
partSet.setAcquisition(acq)
og.toImages(partSet)
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [np.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print("Partset", partFn1)
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partSet, mdFn, alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
convert.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
convert.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
convert.readSetOfParticles(mdFn,
partSet2,
rowToFunc=convert.rowToVolume,
alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print("-" * 5)
print(img.getFileName(), img.getIndex())
print('m1:\n', m1, convert.geometryFromMatrix(m1, False))
print('m2:\n', m2, convert.geometryFromMatrix(m2, False))
self.assertTrue(np.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
if CLEAN_IMAGES:
cleanPath(outputFn)
def convertInputStep(self, movId, partId, postId):
inputMovies = self.inputMovies.get()
inputParts = self.inputParticles.get()
imgStar = self._getFileName('input_particles')
inputPartsFolder = self._getInputPath('particles')
pwutils.makePath(inputPartsFolder)
self.info("Converting set from '%s' into '%s'" %
(inputParts.getFileName(), imgStar))
tableGeneral = Table(columns=[
'rlnImageSizeX', 'rlnImageSizeY', 'rlnImageSizeZ',
'rlnMicrographMovieName', 'rlnMicrographBinning',
'rlnMicrographOriginalPixelSize', 'rlnMicrographDoseRate',
'rlnMicrographPreExposure', 'rlnVoltage',
'rlnMicrographStartFrame', 'rlnMotionModelVersion',
'rlnMicrographGainName', 'rlnMicrographDefectFile'
])
tableShifts = Table(columns=[
'rlnMicrographFrameNumber', 'rlnMicrographShiftX',
'rlnMicrographShiftY'
])
tableCoeffs = Table(
columns=['rlnMotionModelCoeffsIdx', 'rlnMotionModelCoeff'])
# Create the first row, later only the movieName will be updated
xdim, ydim, ndim = inputMovies.getDim()
acq = inputMovies.getAcquisition()
firstMovie = inputMovies.getFirstItem()
a0, aN = firstMovie.getAlignment().getRange()
moviesPixelSize = inputMovies.getSamplingRate()
binningFactor = inputParts.getSamplingRate() / moviesPixelSize
og = convert.OpticsGroups.fromImages(inputMovies)
writer = convert.createWriter(optics=og)
writer.writeSetOfMicrographs(inputMovies,
self._getFileName('input_mics'),
postprocessImageRow=self._updateMic)
tableGeneral.addRow(xdim, ydim, ndim, 'movieName',
binningFactor, moviesPixelSize,
acq.getDosePerFrame(), acq.getDoseInitial(),
acq.getVoltage(), a0, 0, '""', '""')
row = tableGeneral[0]
for movie in inputMovies:
movieStar = self._getMovieStar(movie)
ogId = movie.getAttributeValue('_rlnOpticsGroup', 1)
gainFn = og[ogId].get('rlnMicrographGainName', None)
defectFn = og[ogId].get('rlnMicrographDefectFile', None)
with open(movieStar, 'w') as f:
coeffs = json.loads(
movie.getAttributeValue('_rlnMotionModelCoeff', '[]'))
motionMode = 1 if coeffs else 0
# Update some params in the general table
replaceDict = {
'rlnMicrographMovieName': movie.getFileName(),
'rlnMotionModelVersion': motionMode
}
if gainFn:
replaceDict['rlnMicrographGainName'] = gainFn
if defectFn:
replaceDict['rlnMicrographDefectFile'] = defectFn
tableGeneral[0] = row._replace(**replaceDict)
tableGeneral.writeStar(f, tableName='general', singleRow=True)
# Write shifts
tableShifts.clearRows()
alignment = movie.getAlignment()
shiftsX, shiftsY = alignment.getShifts()
a0, aN = alignment.getRange()
empty = -9999.000
for i in range(1, a0):
tableShifts.addRow(i, empty, empty)
# Adjust the shifts to be relative to the first frame
# so let's add the opposite value
xoff, yoff = -shiftsX[0], -shiftsY[0]
for i in range(a0, aN + 1):
tableShifts.addRow(i, shiftsX[i - a0] + xoff,
shiftsY[i - a0] + yoff)
for i in range(aN + 1, ndim + 1):
tableShifts.addRow(i, empty, empty)
tableShifts.writeStar(f, tableName='global_shift')
# Write coefficients
tableCoeffs.clearRows()
if coeffs:
for i, c in enumerate(coeffs):
tableCoeffs.addRow(i, c)
tableCoeffs.writeStar(f, tableName='local_motion_model')
convert.writeSetOfParticles(inputParts,
imgStar,
outputDir=inputPartsFolder,
alignType=ALIGN_PROJ,
fillMagnification=True)
def writeCtfStarStep(self):
pwutils.cleanPath(self._getExportPath())
pwutils.makePath(self._getExportPath())
inputCTF = self.inputCTF.get()
if self.micrographSource == 0: # same as CTF estimation
ctfMicSet = inputCTF.getMicrographs()
else:
ctfMicSet = self.inputMicrographs.get()
micSet = SetOfMicrographs(filename=':memory:')
psd = inputCTF.getFirstItem().getPsdFile()
hasPsd = psd and os.path.exists(psd)
if hasPsd:
psdPath = self._getExportPath('PSD')
pwutils.makePath(psdPath)
print("Writing PSD files to %s" % psdPath)
for ctf in inputCTF:
# Get the corresponding micrograph
mic = ctfMicSet[ctf.getObjId()]
if mic is None:
print("Skipping CTF id: %s, it is missing from input "
"micrographs. " % ctf.getObjId())
continue
micFn = mic.getFileName()
if not os.path.exists(micFn):
print("Skipping micrograph %s, it does not exists. " % micFn)
continue
mic2 = mic.clone()
mic2.setCTF(ctf)
if hasPsd:
psdFile = ctf.getPsdFile()
newPsdFile = os.path.join(
psdPath,
'%s_psd.mrc' % pwutils.removeExt(mic.getMicName()))
if not os.path.exists(psdFile):
print("PSD file %s does not exits" % psdFile)
print("Skipping micrograph %s" % micFn)
continue
pwutils.copyFile(psdFile, newPsdFile)
# PSD path is relative to Export dir
newPsdFile = os.path.relpath(newPsdFile, self._getExportPath())
ctf.setPsdFile(newPsdFile)
else:
# remove pointer to non-existing psd file
ctf.setPsdFile(None)
micSet.append(mic2)
print("Writing set: %s to: %s" % (inputCTF, self._getStarFile()))
micDir = self._getExportPath('Micrographs')
pwutils.makePath(micDir)
starWriter = convert.createWriter(rootDir=self._getExportPath(),
outputDir=micDir,
useBaseName=True)
starWriter.writeSetOfMicrographs(micSet, self._getStarFile())