def _fillClasses(self, outputClasses):
""" Create the SetOfClasses2D """
inputSet = self.inputClasses.get().getImages()
myRep = md.MetaData('classes@' + self._getExtraPath(
'final_classes.xmd'))
for row in md.iterRows(myRep):
fn = row.getValue(md.MDL_IMAGE)
rep = Particle()
rep.setLocation(xmippToLocation(fn))
repId = row.getObjId()
newClass = Class2D(objId=repId)
newClass.setAlignment2D()
newClass.copyInfo(inputSet)
newClass.setAcquisition(inputSet.getAcquisition())
newClass.setRepresentative(rep)
outputClasses.append(newClass)
i=1
mdBlocks = md.getBlocksInMetaDataFile(self._getExtraPath(
'final_classes.xmd'))
for block in mdBlocks:
if block.startswith('class00'):
mdClass = md.MetaData(block + "@" + self._getExtraPath(
'final_classes.xmd'))
imgClassId = i
newClass = outputClasses[imgClassId]
newClass.enableAppend()
for row in md.iterRows(mdClass):
part = rowToParticle(row)
newClass.append(part)
i+=1
newClass.setAlignment2D()
outputClasses.update(newClass)
def readSetOfParticles(self, starFile, partSet, **kwargs):
""" Convert a star file into a set of particles.
Params:
starFile: the filename of the star file
partsSet: output particles set
Keyword Arguments:
blockName: The name of the data block (default particles)
alignType: alignment type
removeDisabled: Remove disabled items
"""
self._preprocessImageRow = kwargs.get('preprocessImageRow', None)
self._alignType = kwargs.get('alignType', ALIGN_NONE)
self._postprocessImageRow = kwargs.get('postprocessImageRow', None)
self._optics = OpticsGroups.fromStar(starFile)
self._pixelSize = getattr(self._optics.first(), 'rlnImagePixelSize',
1.0)
self._invPixelSize = 1. / self._pixelSize
partsReader = Table.Reader(starFile, tableName='particles')
firstRow = partsReader.getRow()
self._setClassId = hasattr(firstRow, 'rlnClassNumber')
self._setCtf = partsReader.hasAllColumns(self.CTF_LABELS[:3])
particle = Particle()
if self._setCtf:
particle.setCTF(CTFModel())
self._setAcq = kwargs.get("readAcquisition", True)
acq = Acquisition()
acq.setMagnification(kwargs.get('magnification', 10000))
extraLabels = kwargs.get('extraLabels', []) + PARTICLE_EXTRA_LABELS
self.createExtraLabels(particle, firstRow, extraLabels)
self._rowToPart(firstRow, particle)
partSet.setSamplingRate(self._pixelSize)
partSet.setAcquisition(acq)
self._optics.toImages(partSet)
partSet.append(particle)
for row in partsReader:
self._rowToPart(row, particle)
partSet.append(particle)
partSet.setHasCTF(self._setCtf)
partSet.setAlignment(self._alignType)
def getParticleList(start=1, end=3):
""" Mocks a Particle dict"""
newPartDict = {}
for index in range(start, end + 1):
part = Particle()
coord = Coordinate()
coord.setMicId(index)
coord.setMicName(getMicNameFromId(index))
part.setCoordinate(coord)
newPartDict[index] = part
return newPartDict
def createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(
Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
#defocus=15000 + 5000* random.random()
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i] #+500.
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(Transform(a))
self.partSet.append(p)
self.partSet.write()
def test_mrcsLink(self):
""" In this case just a link with .mrcs extension
should be created """
print(magentaStr("\n==> Testing relion - link mrc stack to mrcs:"))
stackFile = self.dsEmx.getFile('particles/particles.mrc')
partSet = SetOfParticles(filename=':memory:')
for i in range(1, 10):
particle = Particle()
particle.setLocation(i, stackFile)
partSet.append(particle)
outputDir = self.getOutputPath()
filesDict = convert.convertBinaryFiles(partSet, outputDir)
print(filesDict)
def test_hdfToStk(self):
""" In this case the hdf stack files should be converted
to .stk spider files for Relion.
"""
print(magentaStr("\n==> Testing relion - convert hdf files to mrcs:"))
stackFiles = [
'BPV_1386_ptcls.hdf', 'BPV_1387_ptcls.hdf', 'BPV_1388_ptcls.hdf'
]
partSet = SetOfParticles(filename=':memory:')
for fn in stackFiles:
particle = Particle()
particle.setLocation(1, self.ds.getFile('particles/%s' % fn))
partSet.append(particle)
outputDir = self.getOutputPath()
filesDict = convert.convertBinaryFiles(partSet, outputDir)
partSet.close()
print(filesDict)
def createOutputStep(self):
particles = self.inputParticles.get()
# Generate the SetOfAlignmet
alignedSet = self._createSetOfParticles()
alignedSet.copyInfo(particles)
inputMd = self._getPath('aligned_particles.xmd')
alignedSet.copyItems(particles,
updateItemCallback=self._updateItem,
itemDataIterator=md.iterRows(inputMd, sortByLabel=md.MDL_ITEM_ID))
# Remove alignment 2D
alignedSet.setAlignment(ALIGN_NONE)
# Define the output average
avgFile = self._getExtraPath("average.xmp")
imgh = ImageHandler()
avgImage = imgh.computeAverage(alignedSet)
avgImage.write(avgFile)
avg = Particle()
avg.setLocation(1, avgFile)
avg.copyInfo(alignedSet)
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.inputParticles, avg)
self._defineOutputs(outputParticles=alignedSet)
self._defineSourceRelation(self.inputParticles, alignedSet)
def createOutputStep(self):
outputStk = self._getFileName('particlesAligned')
if not exists(outputStk):
raise Exception('Output stack %s not produced. ' % outputStk)
particles = self.inputParticles.get()
# Create the output average image
avg = Particle()
avg.copyInfo(self.inputParticles.get())
avg.setLocation(NO_INDEX, self.getAverage())
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.inputParticles, avg)
imgSet = self._createSetOfParticles()
imgSet.copyInfo(particles)
imgSet.setAlignment2D()
imgSet.copyItems(particles,
updateItemCallback=self._updateItem,
itemDataIterator=iter(
range(1,
particles.getSize() + 1)))
self._defineOutputs(outputParticles=imgSet)
self._defineTransformRelation(self.inputParticles, imgSet)
def createOutputStep(self):
""" Store the setOfParticles object
as result of the protocol.
"""
particles = self.inputParticles.get()
# Define the output average
avgFile = self._getExtraPath('level_00', 'class_classes.stk')
avg = Particle()
avg.setLocation(1, avgFile)
avg.copyInfo(particles)
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.inputParticles, avg)
# Generate the Set of Particles with alignment
alignedSet = self._createSetOfParticles()
alignedSet.copyInfo(particles)
alignedSet.setRepresentative(avg)
alignedSet.copyItems(particles,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(
self.imgsFn, sortByLabel=md.MDL_ITEM_ID))
alignedSet.setAlignment(ALIGN_2D)
self._defineOutputs(outputParticles=alignedSet)
self._defineSourceRelation(self.inputParticles, alignedSet)
def _preprocessClass(self, classItem, classRow):
classItem.average = Particle()
classItem.average.setLocation(
xmippToLocation(classRow.getValue(emlib.MDL_IMAGE2)))
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
given the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print("\n")
print("*" * 80)
print("* Launching test: ", fileKey)
print("*" * 80)
is2D = alignType == ALIGN_2D
if fileKey == 'alignShiftRotExp':
# relion requires mrcs stacks
origFn = self.dataset.getFile(fileKey)
stackFn = replaceExt(origFn, ".mrcs")
createLink(origFn, stackFn)
else:
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile("reconstruction/gold/" + fileKey +
'_Gold_rln_output.vol')
if PRINT_FILES:
print("BINARY DATA: ", stackFn)
print("SET1: ", partFn1)
print(" MD: ", mdFn)
print("SET2: ", partFn2)
print("OUTPUT: ", outputFn)
print("GOLD: ", goldFn)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
acq = Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000)
og = OpticsGroups.create(rlnMtfFileName="mtfFile1.star",
rlnImageSize=128)
partSet.setSamplingRate(1.0)
partSet.setAcquisition(acq)
og.toImages(partSet)
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [np.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print("Partset", partFn1)
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partSet, mdFn, alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
convert.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
convert.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
convert.readSetOfParticles(mdFn,
partSet2,
rowToFunc=convert.rowToVolume,
alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print("-" * 5)
print(img.getFileName(), img.getIndex())
print('m1:\n', m1, convert.geometryFromMatrix(m1, False))
print('m2:\n', m2, convert.geometryFromMatrix(m2, False))
self.assertTrue(np.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
if CLEAN_IMAGES:
cleanPath(outputFn)
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = Particle()
for mic in micList:
# We need to make this dict because there is no ID in the .xmd file
coordDict = {}
for coord in self.coordDict[mic.getObjId()]:
pos = self._getPos(coord)
if pos in coordDict:
print(
"WARNING: Ignoring duplicated coordinate: %s, id=%s" %
(coord.getObjId(), pos))
coordDict[pos] = coord
added = set() # Keep track of added coords to avoid duplicates
fnMicXmd = self._getMicXmd(mic)
if exists(fnMicXmd):
for row in md.iterRows(fnMicXmd):
pos = (row.getValue(md.MDL_XCOOR),
row.getValue(md.MDL_YCOOR))
coord = coordDict.get(pos, None)
if coord is not None and coord.getObjId() not in added:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
p.setLocation(
xmippToLocation(row.getValue(md.MDL_IMAGE)))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
# adding the variance and Gini coeff. value of the mic zone
setXmippAttributes(p, row, md.MDL_SCORE_BY_VAR)
setXmippAttributes(p, row, md.MDL_SCORE_BY_GINI)
if row.containsLabel(md.MDL_ZSCORE_DEEPLEARNING1):
setXmippAttributes(p, row,
md.MDL_ZSCORE_DEEPLEARNING1)
# disabled particles (in metadata) should not add to the
# final set
if row.getValue(md.MDL_ENABLED) > 0:
outputParts.append(p)
added.add(coord.getObjId())
# Release the list of coordinates for this micrograph since it
# will not be longer needed
del self.coordDict[mic.getObjId()]
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
relionToLocation = relion.convert.relionToLocation
p = Particle()
p._rlnOpticsGroup = Integer()
acq = self.getInputMicrographs().getAcquisition()
# JMRT: Ideally I would like to disable the whole Acquisition for each
# particle row, but the SetOfImages will set it again.
# Another option could be to disable in the set, but then in
# streaming, other protocols might get the wrong optics info
pAcq = Acquisition(magnification=acq.getMagnification(),
voltage=acq.getVoltage(),
amplitudeContrast=acq.getAmplitudeContrast(),
sphericalAberration=acq.getSphericalAberration())
p.setAcquisition(pAcq)
tmp = self._getTmpPath()
extra = self._getExtraPath()
for mic in micList:
posSet = set()
coordDict = {self._getPos(c): c
for c in self.coordDict[mic.getObjId()]}
del self.coordDict[mic.getObjId()]
ogNumber = mic.getAttributeValue('_rlnOpticsGroup', 1)
partsStar = self.__getMicFile(mic, '_extract.star', folder=tmp)
partsTable = relion.convert.Table(fileName=partsStar)
stackFile = self.__getMicFile(mic, '.mrcs', folder=tmp)
endStackFile = self.__getMicFile(mic, '.mrcs', folder=extra)
pwutils.moveFile(stackFile, endStackFile)
for part in partsTable:
pos = (int(float(part.rlnCoordinateX)),
int(float(part.rlnCoordinateY)))
if pos in posSet:
print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
coord = None
else:
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(part.rlnImageName)
p.setLocation(idx, endStackFile)
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
p._rlnOpticsGroup.set(ogNumber)
outputParts.append(p)
posSet.add(pos)
def createOutputStep(self):
input = self.input.get()
imgSetOut = self._createSetOfParticles()
imgSetOut.setSamplingRate(input.getSamplingRate())
imgSetOut.setAlignmentProj()
for i, subtomo in enumerate(input.iterItems()):
idx = subtomo.getObjId()
p = Particle()
p.setLocation(ih._convertToLocation((i+1, self._getExtraPath("projections.mrcs"))))
p._subtomogramID = String(idx)
if type(subtomo) == SubTomogram:
if subtomo.hasCoordinate3D():
coord = Coordinate()
coord.setX(subtomo.getCoordinate3D().getX(const.BOTTOM_LEFT_CORNER))
coord.setY(subtomo.getCoordinate3D().getY(const.BOTTOM_LEFT_CORNER))
p.setCoordinate(coord)
p.setClassId(subtomo.getClassId())
if subtomo.hasTransform():
transform = Transform()
transform.setMatrix(subtomo.getTransform().getMatrix())
p.setTransform(transform)
imgSetOut.append(p)
imgSetOut.setObjComment(self.getSummary(imgSetOut))
self._defineOutputs(outputParticles=imgSetOut)
self._defineSourceRelation(self.input, imgSetOut)
if self.radAvg.get():
avgFile = self._getExtraPath("average.xmp")
imgh = ih()
avgImage = imgh.computeAverage(imgSetOut)
avgImage.write(avgFile)
avg = Particle()
avg.setLocation(1, avgFile)
avg.copyInfo(imgSetOut)
self._defineOutputs(outputAverage=avg)
self._defineSourceRelation(self.input, avg)
def particlesFactory(index):
newParticle = Particle(location=(1, self.particlesStk))
newParticle.setCoordinate(Coordinate(x=1, y=2))
newParticle.setAcquisition(Acquisition())
return newParticle
def particlesFactory(index):
newParticle = Particle(location=(1, self.particlesStk))
return newParticle