def _testInv(matrix):
matrix = np.array(matrix)
a = Transform(matrix)
row1 = md.Row()
convert.alignmentToRow(a, row1, alignType=ALIGN_2D)
row2 = None
row3 = md.Row()
convert.alignmentToRow(a, row3, alignType=ALIGN_PROJ)
return row1, row2, row3
def _testInv(matrix):
matrix = np.array(matrix)
a = Transform(matrix)
row = metadata.Row()
alignmentToRow(a, row, alignType=ALIGN_2D)
row2 = metadata.Row()
alignmentToRow(a, row2, alignType=ALIGN_3D)
row3 = metadata.Row()
alignmentToRow(a, row3, alignType=ALIGN_PROJ)
return row, row2, row3
def alignParticlesStep(self):
fhInputTranMat = self._getExtraPath('transformation-matrix.txt')
outParticlesFn = self._getExtraPath('outputParticles.xmd')
transMatFromFile = np.loadtxt(fhInputTranMat)
transformationMat = np.reshape(transMatFromFile, (4, 4))
transform = Transform()
transform.setMatrix(transformationMat)
resultMat = Transform()
outputParts = md.MetaData()
mdToAlign = md.MetaData(self.imgsInputFn)
for row in md.iterRows(mdToAlign):
inMat = rowToAlignment(row, ALIGN_PROJ)
partTransformMat = inMat.getMatrix()
partTransformMatrix = np.matrix(partTransformMat)
newTransformMatrix = np.matmul(transformationMat,
partTransformMatrix)
resultMat.setMatrix(newTransformMatrix)
rowOut = md.Row()
rowOut.copyFromRow(row)
alignmentToRow(resultMat, rowOut, ALIGN_PROJ)
rowOut.addToMd(outputParts)
outputParts.write(outParticlesFn)
cleanPath(self.imgsInputFn)
def createAngDistributionSqlite(self, sqliteFn, numberOfParticles,
itemDataIterator):
if not os.path.exists(sqliteFn):
# List of list of 3 elements containing angleTilt, anglePsi, weight
projectionList = []
def getCloseProjection(angleRot, angleTilt):
""" Get an existing projection close to angleRot, angleTilt.
Return None if not found close enough.
"""
for projection in projectionList:
if (abs(projection[0] - angleRot) <= 0.5
and abs(projection[1] - angleTilt) <= 0.5):
return projection
return None
weight = 1. / numberOfParticles
for angleRot, angleTilt in itemDataIterator:
projection = getCloseProjection(angleRot, angleTilt)
if projection is None:
projectionList.append([angleRot, angleTilt, weight])
else:
projection[2] = projection[2] + weight
mdProj = md.MetaData()
for projection in projectionList:
mdRow = md.Row()
mdRow.setValue(md.MDL_ANGLE_ROT, projection[0])
mdRow.setValue(md.MDL_ANGLE_TILT, projection[1])
mdRow.setValue(md.MDL_WEIGHT, projection[2])
mdRow.writeToMd(mdProj, mdProj.addObject())
mdProj.write(sqliteFn)
def mergeVolumesStep(self, numOfRuns):
mdOut = md.MetaData()
volFnList = []
clsDistList = []
accList = []
for run in range(1, numOfRuns + 1):
it = self._lastIter(run)
modelFile = self._getFileName('model', ruNum=run, iter=it)
mdIn = md.MetaData('model_classes@%s' % modelFile)
for row in md.iterRows(mdIn, md.RLN_MLMODEL_REF_IMAGE):
volFn = row.getValue(md.RLN_MLMODEL_REF_IMAGE)
clsDist = row.getValue('rlnClassDistribution')
accRot = row.getValue('rlnAccuracyRotations')
if accRot <= 90:
volFnList.append(volFn)
clsDistList.append(clsDist)
accList.append(accRot)
self.std = np.std(accList)
score = self._estimateScore(accList, clsDistList)
tupleList = [(fn, s) for fn, s in zip(volFnList, score)]
nVols = self.numOfVols.get()
sortList = sorted(tupleList, reverse=True, key=lambda x: x[1])[0:nVols]
row = md.Row()
for val in sortList:
fn, score = val
row.setValue(md.RLN_MLMODEL_REF_IMAGE, fn)
row.addToMd(mdOut)
mdOut.write(self._getRefStar())
def correlationCudaStep(self, thIdx, gpuId, totalGpu):
mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc))
self.numCones = mdNumCones.size()
# Cuda Correlation step - creating the metadata
predCones = np.loadtxt(self._getConePrediction())
mdConeList = []
numMax = int(self.numConesSelected)
for i in range(self.numCones):
mdConeList.append(emlib.MetaData())
mdIn = emlib.MetaData(self.imgsFn)
for i in range(self.numCones):
idx = i + 1
if (idx % totalGpu) != thIdx:
continue
#modelFn = 'modelCone%d_aux' % idx
#f = open(join(self._getExtraPath(), modelFn+'.txt'),'r')
#mae = float(f.readline())
#f.close()
modelFn = 'modelCone%d' % idx
positions = []
for n in range(numMax):
posAux = np.where(predCones[:, (n * 2) + 1] == (i + 1))
positions = positions + (np.ndarray.tolist(posAux[0]))
if len(positions) > 0 and exists(self._getExtraPath(modelFn + '.h5')):
print("Classifying cone ", idx, "in GPU ", gpuId)
for pos in positions:
id = pos + 1
row = md.Row()
row.readFromMd(mdIn, id)
row.addToMd(mdConeList[i])
fnExpCone = self._getExtraPath('metadataCone%d.xmd' % (i + 1))
mdConeList[i].write(fnExpCone)
fnProjCone = self._getExtraPath('projectionsCudaCorr%d.xmd' % (i + 1))
self.runJob("xmipp_metadata_utilities", "-i %s --fill ref lineal 1 1 " % (fnProjCone), numberOfMpi=1)
fnOutCone = self._getOutCone(i + 1)
if not exists(self._getExtraPath(fnOutCone)):
params = ' -i %s' % fnExpCone
params += ' -r %s' % fnProjCone
params += ' -o %s' % self._getExtraPath(fnOutCone)
params += ' --dev %s '%(gpuId)
self.runJob("xmipp_cuda_align_significant", params, numberOfMpi=1)
def _convertRef(self):
ih = ImageHandler()
inputObj = self.inputVolumes.get()
row = md.Row()
refMd = md.MetaData()
for vol in inputObj:
newVolFn = self._convertVol(ih, vol)
row.setValue(md.RLN_MLMODEL_REF_IMAGE, newVolFn)
row.addToMd(refMd)
refMd.write(self._getRefStar())
def test_rowToImage(self):
row = metadata.Row()
index = 1
filename = 'images.stk'
row.setValue(emlib.MDL_ITEM_ID, 1)
row.setValue(emlib.MDL_IMAGE, '%d@%s' % (index, filename))
img = rowToImage(row, emlib.MDL_IMAGE, Particle)
# Check correct index and filename
self.assertEquals(index, img.getIndex())
self.assertEquals(filename, img.getFileName())
def createRefMd(self, vols):
refVols = self._getExtraPath('ref_volumes.xmd')
mdFn = md.MetaData()
if self.isSetOfVolumes():
for vol in vols:
imgId = vol.getObjId()
row = md.Row()
row.setValue(md.MDL_ITEM_ID, int(imgId))
row.setValue(md.MDL_IMAGE, getImageLocation(vol))
row.setValue(md.MDL_ENABLED, 1)
row.addToMd(mdFn)
else:
imgId = vols.getObjId()
row = md.Row()
row.setValue(md.MDL_ITEM_ID, int(imgId))
row.setValue(md.MDL_IMAGE, getImageLocation(vols))
row.setValue(md.MDL_ENABLED, 1)
row.addToMd(mdFn)
mdFn.write(refVols, md.MD_APPEND)
def createInputMd(self, vols):
fnVols = self._getExtraPath('input_volumes.xmd')
if self.copyAlignment:
alignType = vols.getAlignment()
else:
alignType = ALIGN_NONE
writeSetOfVolumes(vols, fnVols,
postprocessImageRow=self._postprocessVolumeRow,
alignType=alignType)
if not vols.hasAlignment() or not self.copyAlignment:
mdFn = md.MetaData(fnVols)
mdFn.fillConstant(md.MDL_ANGLE_ROT, 0.)
mdFn.fillConstant(md.MDL_ANGLE_TILT, 0.)
mdFn.fillConstant(md.MDL_ANGLE_PSI, 0.)
mdFn.fillConstant(md.MDL_SHIFT_X, 0.)
mdFn.fillConstant(md.MDL_SHIFT_Y, 0.)
mdFn.fillConstant(md.MDL_SHIFT_Z, 0.)
mdFn.write(fnVols, md.MD_OVERWRITE)
# set missing angles
missType = ['wedge_y', 'wedge_x', 'pyramid', 'cone']
missNum = self.missingDataType.get()
missAng = self.missingAng.get()
missDataFn = self._getExtraPath('wedges.xmd')
missAngValues = str(missAng).strip().split()
thetaY0, thetaYF, thetaX0, thetaXF = 0, 0, 0, 0
mdFn = md.MetaData()
if missNum == MISSING_WEDGE_X:
thetaX0, thetaXF = missAngValues
elif missNum == MISSING_WEDGE_Y:
thetaY0, thetaYF = missAngValues
elif missNum == MISSING_PYRAMID:
thetaY0, thetaYF, thetaX0, thetaXF = missAngValues
else: # MISSING_CONE
thetaY0 = missAngValues[0]
for i in range(1, vols.getSize() + 1):
row = md.Row()
row.setValue(md.MDL_MISSINGREGION_NR, missNum + 1)
row.setValue(md.MDL_MISSINGREGION_TYPE, missType[missNum])
row.setValue(md.MDL_MISSINGREGION_THX0, float(thetaX0))
row.setValue(md.MDL_MISSINGREGION_THXF, float(thetaXF))
row.setValue(md.MDL_MISSINGREGION_THY0, float(thetaY0))
row.setValue(md.MDL_MISSINGREGION_THYF, float(thetaYF))
row.addToMd(mdFn)
mdFn.write(missDataFn, md.MD_APPEND)
def test_writeSetOfDefocusGroups(self):
#TODO: FIX THIS test according to the new SetOfDefocusGroup
return
#reference metadata
md = emlib.MetaData()
objId = md.addObject()
defocusGroupRow = metadata.Row()
defocusGroupRow.setValue(emlib.MDL_ENABLED, 1)
defocusGroupRow.setValue(emlib.MDL_CTF_GROUP, 1)
defocusGroupRow.setValue(emlib.MDL_MIN, 2000.)
defocusGroupRow.setValue(emlib.MDL_MAX, 2500.)
defocusGroupRow.setValue(emlib.MDL_AVG, 2100.)
defocusGroupRow.writeToMd(md, objId)
objId = md.addObject()
defocusGroupRow.setValue(emlib.MDL_ENABLED, 1)
defocusGroupRow.setValue(emlib.MDL_CTF_GROUP, 2)
defocusGroupRow.setValue(emlib.MDL_MIN, 3000.)
defocusGroupRow.setValue(emlib.MDL_MAX, 5500.)
defocusGroupRow.setValue(emlib.MDL_AVG, 5000.)
defocusGroupRow.writeToMd(md, objId)
#
fnScipion = self.getOutputPath("writeSetOfDefocusGroups.sqlite")
setOfDefocus = SetOfDefocusGroup(filename=fnScipion)
df = DefocusGroup()
df.setDefocusMin(2000.)
df.setDefocusMax(2500.)
df.setDefocusAvg(2100.)
setOfDefocus.append(df)
df.cleanObjId()
df.setDefocusMin(3000)
df.setDefocusMax(5500)
df.setDefocusAvg(5000)
setOfDefocus.append(df)
fnXmipp = self.getOutputPath("writeSetOfDefocusGroups.xmd")
fnScipion = self.getOutputPath("writeSetOfDefocusGroups2.xmd")
writeSetOfDefocusGroups(setOfDefocus, fnXmipp)
mdAux = emlib.MetaData(fnXmipp)
md.write(fnScipion)
print("Comparing metadatas: \n%s\n%s" % (fnXmipp, fnScipion))
self.assertEqual(md, mdAux, "test writeSetOfDefocusGroups fails")
def test_rowToCtfModel(self):
row = metadata.Row()
row.setValue(emlib.MDL_CTF_DEFOCUSU, 2520.)
row.setValue(emlib.MDL_CTF_DEFOCUSV, 2510.)
row.setValue(emlib.MDL_CTF_DEFOCUS_ANGLE, 45.)
ctf = rowToCtfModel(row)
# Check that the ctf object was properly set
self.assertTrue(ctf.equalAttributes(self.getCTF(2520., 2510., 45.)))
# Check when the EMX standarization takes place
row.setValue(emlib.MDL_CTF_DEFOCUSV, 2530.)
ctf = rowToCtfModel(row)
self.assertTrue(ctf.equalAttributes(self.getCTF(2530., 2520., 135.)))
# When one of CTF labels is missing, None should be returned
row.removeLabel(emlib.MDL_CTF_DEFOCUSV)
ctf = rowToCtfModel(row)
self.assertIsNone(ctf)
def _convertRef(self):
ih = ImageHandler()
inputObj = self.inputVolumes.get()
subset = SetOfVolumes(filename=":memory:")
refMd = md.MetaData()
for vol in inputObj.iterItems(orderBy='RANDOM()'):
subset.append(vol)
subsetSize = self.numOfVols.get()
minSize = min(subsetSize, inputObj.getSize())
if subset.getSize() <= minSize:
row = md.Row()
newVolFn = self._convertVol(ih, vol)
row.setValue(md.RLN_MLMODEL_REF_IMAGE, newVolFn)
row.addToMd(refMd)
else:
break
refMd.write(self._getRefStar())
def _writeXmippPosFile(self, movie, coordinatesName, shiftX, shiftY):
""" Create Xmipp coordinate files to be extracted
from the frames of the movie.
"""
coordSet = self.getCoords()
mData = md.MetaData()
coordRow = md.Row()
for coord in coordSet.iterCoordinates(movie.getObjId()):
coord.shiftX(int(round(float(shiftX))))
coord.shiftY(int(round(float(shiftY))))
coordinateToRow(coord, coordRow)
coordRow.writeToMd(mData, mData.addObject())
self.info("Writing coordinates metadata: %s, with shifts: %s %s" %
(coordinatesName, shiftX, shiftY))
mData.write('particles@' + coordinatesName)
def generateOutputClasses(self, classesOut, firstTime):
if firstTime:
self._saveFileDataClasses(classesOut,
self._getExtraPath('last_classes.xmd'))
# Add the two new classes
for i in range(1, 3):
mdImgsInClass = md.MetaData('class%06d_images@%s' %
(i, classesOut))
mdImgsInClass.write(self._getExtraPath('dataClass%06d.xmd' %
i))
return
finalMetadata = self._getExtraPath('aux_classes.stk')
lastMetadata = self._getExtraPath('last_classes.xmd')
newMetadata = classesOut
total = self.averageClasses(finalMetadata, lastMetadata, newMetadata,
False)
copy(self._getExtraPath('aux_classes.stk'),
self._getExtraPath('last_classes.stk'))
mdAll = md.MetaData()
newRef = 1
for i in total:
if i != 0:
row = md.Row()
row.setValue(md.MDL_REF, newRef)
row.setValue(md.MDL_IMAGE, '%06d@' % newRef + finalMetadata)
row.setValue(md.MDL_CLASS_COUNT, i)
row.addToMd(mdAll)
newRef += 1
mdAll.write('classes@' + finalMetadata[:-3] + 'xmd', MD_APPEND)
copy(self._getExtraPath('aux_classes.xmd'),
self._getExtraPath('last_classes.xmd'))
newRef = 1
for i, val in enumerate(total):
if val != 0:
self._unionDataClass(classesOut, i + 1, newRef)
newRef += 1
def correlationSignificantStep(self):
mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc))
self.numCones = mdNumCones.size()
# Cuda Correlation step - creating the metadata
predCones = np.loadtxt(self._getConePrediction())
mdConeList = []
numMax = int(self.numConesSelected)
for i in range(self.numCones):
mdConeList.append(emlib.MetaData())
mdIn = emlib.MetaData(self.imgsFn)
for i in range(self.numCones):
print("Classifying cone ", i + 1)
positions = []
for n in range(numMax):
posAux = np.where(predCones[:, (n * 2) + 1] == (i + 1))
positions = positions + (np.ndarray.tolist(posAux[0]))
if len(positions) > 0:
for pos in positions:
id = pos + 1
row = md.Row()
row.readFromMd(mdIn, id)
row.addToMd(mdConeList[i])
fnExpCone = self._getExtraPath('metadataCone%d.xmd' % (i + 1))
mdConeList[i].write(fnExpCone)
fnProjCone = self._getExtraPath('projectionsCudaCorr%d.xmd' % (i + 1))
fnOutCone = self._getOutCone(i + 1)
if not exists(self._getExtraPath(fnOutCone)):
# Correlation step - calling significant program
args = '-i %s --initgallery %s --odir %s --dontReconstruct --useForValidation %d ' \
'--dontCheckMirrors --maxShift 30' % (fnExpCone, fnProjCone, self._getExtraPath(), 1)
self.runJob('xmipp_reconstruct_significant', args,
numberOfMpi=self.myMPI.get())
copy(self._getExtraPath('images_significant_iter001_00.xmd'), self._getExtraPath(fnOutCone))
remove(self._getExtraPath('angles_iter001_00.xmd'))
remove(self._getExtraPath('images_significant_iter001_00.xmd'))
def writeSetOfMicrographs(micSet, filename):
""" Simplified function borrowed from xmipp. """
mdata = md.MetaData()
for img in micSet:
objId = mdata.addObject()
imgRow = md.Row()
imgRow.setValue(md.MDL_ITEM_ID, int(objId))
index, fname = img.getLocation()
fn = ImageHandler.locationToXmipp((index, fname))
imgRow.setValue(md.MDL_MICROGRAPH, fn)
if img.isEnabled():
enabled = 1
else:
enabled = -1
imgRow.setValue(md.MDL_ENABLED, enabled)
imgRow.writeToMd(mdata, objId)
mdata.write('Micrographs@%s' % filename)
def applyAlignmentStep(self, inputFn):
inputSt = self.inputSubtomograms.get()
# Window subtomograms twice their size
windowedStk = self._getExtraPath('windowed_subtomograms.stk')
self.runJob(
'xmipp_transform_window',
'-i %s -o %s --size %d --save_metadata_stack' %
(inputFn, windowedStk, 2 * inputSt.getFirstItem().getDim()[0]),
numberOfMpi=1)
# Add input transform matrix to md generated by xmipp_transform_window
mdWindow = md.MetaData(self._getExtraPath('windowed_subtomograms.xmd'))
mdWindowTransform = md.MetaData()
idList = list(inputSt.getIdSet())
for row in md.iterRows(mdWindow):
rowOut = md.Row()
rowOut.copyFromRow(row)
id = row.getValue(MDL_IMAGE)
id = id.split('@')[0]
id = id.strip('0')
alignmentToRow(inputSt[(idList[int(id) - 1])].getTransform(),
rowOut, pwem.ALIGN_3D)
rowOut.addToMd(mdWindowTransform)
mdWindowTransform.write(
self._getExtraPath("window_with_original_geometry.xmd"))
# Align subtomograms
self.runJob(
'xmipp_transform_geometry', '-i %s -o %s --apply_transform' %
(self._getExtraPath("window_with_original_geometry.xmd"),
self._getExtraPath('aligned_subtomograms.stk')))
# Window subtomograms to their original size
alignStk = self._getExtraPath('aligned_subtomograms.stk')
outputStk = self._getPath('output_subtomograms.stk')
self.runJob('xmipp_transform_window',
'-i %s -o %s --size %d ' %
(alignStk, outputStk,
self.inputSubtomograms.get().getFirstItem().getDim()[0]),
numberOfMpi=1)
return [outputStk]
def _writeMDFinal(self, allInFns, coneFns, coneCCs, mdCones, shiftX, shiftY, fnFinal):
mdFinal = emlib.MetaData()
row = md.Row()
for myFn in allInFns:
myCCs = []
myCones = []
myPos = []
for n in range(self.numCones):
if myFn in coneFns[n]:
pos = coneFns[n].index(myFn)
myPos.append(pos)
if abs(shiftX[n][pos])<30 and abs(shiftY[n][pos])<30:
myCCs.append(coneCCs[n][pos])
else:
myCCs.append(0)
myCones.append(n + 1)
if len(myPos) > 0:
if max(myCCs)==0:
continue
coneMax = myCones[myCCs.index(max(myCCs))]
objId = myPos[myCCs.index(max(myCCs))] + 1
row.readFromMd(mdCones[coneMax - 1], objId)
row.addToMd(mdFinal)
mdFinal.write(fnFinal)
def realignStep(self):
inputMdName = self._getExtraPath('inputClasses.xmd')
writeSetOfClasses2D(self.inputClasses.get(), inputMdName,
writeParticles=True)
centeredStackName = self._getExtraPath('centeredStack.stk')
self._params = {'input': inputMdName,
'output': centeredStackName}
args = ('-i %(input)s -o %(output)s --save_metadata_transform')
self.runJob("xmipp_transform_center_image", args % self._params,
numberOfMpi=1)
centeredMdName = centeredStackName.replace('stk', 'xmd')
centeredMd = md.MetaData(centeredMdName)
centeredStack = md.MetaData(centeredStackName)
listName = []
listTransform=[]
for rowStk in md.iterRows(centeredStack):
listName.append(rowStk.getValue(md.MDL_IMAGE))
for rowMd in md.iterRows(centeredMd):
listTransform.append(rowToAlignment(rowMd, ALIGN_2D))
mdNewClasses = md.MetaData()
for i, row in enumerate(md.iterRows(inputMdName)):
newRow = md.Row()
newRow.setValue(md.MDL_IMAGE, listName[i])
refNum = row.getValue(md.MDL_REF)
newRow.setValue(md.MDL_REF, refNum)
classCount = row.getValue(md.MDL_CLASS_COUNT)
newRow.setValue(md.MDL_CLASS_COUNT, classCount)
newRow.addToMd(mdNewClasses)
mdNewClasses.write('classes@' + self._getExtraPath('final_classes.xmd'),
MD_APPEND)
mdImages = md.MetaData()
i=0
mdBlocks = md.getBlocksInMetaDataFile(inputMdName)
resultMat = Transform()
for block in mdBlocks:
if block.startswith('class00'):
newMat = listTransform[i]
newMatrix = newMat.getMatrix()
mdClass = md.MetaData(block + "@" + inputMdName)
mdNewClass = md.MetaData()
i+=1
for rowIn in md.iterRows(mdClass):
#To create the transformation matrix (and its parameters)
# for the realigned particles
if rowIn.getValue(md.MDL_ANGLE_PSI)!=0:
flag_psi=True
if rowIn.getValue(md.MDL_ANGLE_ROT)!=0:
flag_psi=False
inMat = rowToAlignment(rowIn, ALIGN_2D)
inMatrix = inMat.getMatrix()
resultMatrix = np.dot(newMatrix,inMatrix)
resultMat.setMatrix(resultMatrix)
rowOut=md.Row()
rowOut.copyFromRow(rowIn)
alignmentToRow(resultMat, rowOut, ALIGN_2D)
if flag_psi==False:
newAngle = rowOut.getValue(md.MDL_ANGLE_PSI)
rowOut.setValue(md.MDL_ANGLE_PSI, 0.)
rowOut.setValue(md.MDL_ANGLE_ROT, newAngle)
#To create the new coordinates for the realigned particles
inPoint = np.array([[0.],[0.],[0.],[1.]])
invResultMat = np.linalg.inv(resultMatrix)
centerPoint = np.dot(invResultMat,inPoint)
rowOut.setValue(md.MDL_XCOOR, rowOut.getValue(
md.MDL_XCOOR)+int(centerPoint[0]))
rowOut.setValue(md.MDL_YCOOR, rowOut.getValue(
md.MDL_YCOOR)+int(centerPoint[1]))
rowOut.addToMd(mdNewClass)
mdNewClass.write(block + "@" + self._getExtraPath(
'final_classes.xmd'), MD_APPEND)
mdImages.unionAll(mdNewClass)
mdImages.write(self._getExtraPath('final_images.xmd'))
def rowFromMd(mdata, objId):
row = md.Row()
row.readFromMd(mdata, objId)
return row
def test_forward_backwards(self):
"""convert transformation matrixt to xmipp and back"""
mList = [
[
[0.71461016, 0.63371837, -0.29619813, 1.], #a1
[-0.61309201, 0.77128059, 0.17101008, 2.],
[0.33682409, 0.059391174, 0.93969262, 3.],
[0, 0, 0, 1.]
],
[
[0., 0., -1., 0.], #a2
[0., 1., 0., 0.],
[1., 0., 0., 0.],
[0., 0., 0., 1.]
],
[
[0., 1., 0., 0.], #a3
[0., 0., 1., 0.],
[1., 0., 0., 0.],
[0., 0., 0., 1.]
],
[
[0.22612257, 0.82379508, -0.51983678, 0.], #a4
[-0.88564873, 0.39606407, 0.24240388, 0.],
[0.40557978, 0.40557978, 0.81915206, 0.],
[0., 0., 0., 1.]
],
[
[-0.78850311, -0.24329656, -0.56486255, 0.], #a5
[0.22753462, -0.96866286, 0.099600501, 0.],
[-0.57139379, -0.049990479, 0.81915206, 0.],
[0., 0., 0., 1.]
],
[
[1.0, 0.0, 0.0, 0.0], #a6
[0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0],
[0.0, 0.0, 0.0, 1.0]
],
[
[0., 0., -1., 0.], #a7
[-1., 0., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 0., 1.]
]
]
aList = [np.array(m) for m in mList]
rowa = metadata.Row()
rowap = metadata.Row()
rowb = metadata.Row()
rowb1 = metadata.Row()
rowb2 = metadata.Row()
rowb3 = metadata.Row()
labelList = [
emlib.MDL_ANGLE_ROT, emlib.MDL_ANGLE_TILT, emlib.MDL_ANGLE_PSI,
emlib.MDL_SHIFT_X, emlib.MDL_SHIFT_Y, emlib.MDL_SHIFT_Z
]
for i, a in enumerate(aList):
a = Transform(aList[i])
alignmentToRow(a, rowa, ALIGN_PROJ)
b = rowToAlignment(rowa, ALIGN_PROJ)
alignmentToRow(b, rowb, ALIGN_PROJ)
#same two matrices
self.assertTrue(
np.allclose(a.getMatrix(), b.getMatrix(), rtol=1e-2))
for label in labelList:
auxBtilt = rowb.getValue(label)
auxAtilt = rowa.getValue(label)
#same two rows
self.assertAlmostEqual(auxBtilt,
auxAtilt,
places=3,
msg=None,
delta=None)
rowa.setValue(emlib.MDL_FLIP, True)
aflip = rowToAlignment(rowa, ALIGN_PROJ)
alignmentToRow(aflip, rowap, ALIGN_PROJ)
aflipp = rowToAlignment(rowap, ALIGN_PROJ)
self.assertTrue(
np.allclose(aflip.getMatrix(), aflipp.getMatrix(), rtol=1e-2))
def applyAlignStep(self):
"""Align subtomograms to be in the same orientation as the reference"""
inputSet = self.inputSubtomos.get()
outputFn = self._getExtraPath('input_subtomos.xmd')
if inputSet.getFirstItem().getFileName().endswith('.mrc') or \
inputSet.getFirstItem().getFileName().endswith('.map'):
S = self._createSetOfSubTomograms()
S.setSamplingRate(inputSet.getSamplingRate())
for subtomo in inputSet:
s = subtomo.clone()
s.setFileName(subtomo.getFileName() + ':mrc')
S.append(s)
writeSetOfVolumes(S, outputFn, alignType=ALIGN_3D)
else:
writeSetOfVolumes(inputSet, outputFn, alignType=ALIGN_3D)
# Window subtomograms twice their size
windowedStk = self._getExtraPath('windowed_subtomograms.stk')
self.runJob(
'xmipp_transform_window',
'-i %s -o %s --size %d --save_metadata_stack' %
(outputFn, windowedStk, 2 * inputSet.getFirstItem().getDim()[0]),
numberOfMpi=1)
# Add input transform matrix to md generated by xmipp_transform_window
mdWindow = md.MetaData(self._getExtraPath('windowed_subtomograms.xmd'))
mdWindowTransform = md.MetaData()
idList = list(inputSet.getIdSet())
for row in md.iterRows(mdWindow):
rowOut = md.Row()
rowOut.copyFromRow(row)
id = row.getValue(MDL_IMAGE)
id = id.split('@')[0]
id = id.strip('0')
alignmentToRow(inputSet[(idList[int(id) - 1])].getTransform(),
rowOut, ALIGN_3D)
rowOut.addToMd(mdWindowTransform)
mdWindowTransform.write(
self._getExtraPath("window_with_original_geometry.xmd"))
# Align subtomograms
self.runJob(
'xmipp_transform_geometry',
'-i %s -o %s --apply_transform --dont_wrap' %
(self._getExtraPath("window_with_original_geometry.xmd"),
self._getExtraPath('aligned_subtomograms.stk')))
# Window subtomograms to their original size
alignStk = self._getExtraPath('aligned_subtomograms.stk')
outputStk = self._getExtraPath('output_subtomograms.stk')
self.runJob('xmipp_transform_window',
'-i %s -o %s --size %d ' %
(alignStk, outputStk, inputSet.getFirstItem().getDim()[0]),
numberOfMpi=1)
self.alignedSet = self._createSetOfSubTomograms()
self.alignedSet.copyInfo(inputSet)
inputMd = self._getExtraPath('output_subtomograms.stk')
self.alignedSet.copyItems(inputSet,
updateItemCallback=self._updateItemAlign,
itemDataIterator=md.iterRows(
inputMd, sortByLabel=md.MDL_ITEM_ID))
def convertInputStep(self, classesFn):
inputSet = self.inputSet.get()
if isinstance(inputSet, SetOfClasses2D):
writeSetOfClasses2D(inputSet, classesFn, writeParticles=False)
else:
writeSetOfParticles(inputSet, classesFn)
# To re-sample input images
fnDir = self._getExtraPath()
fnNewParticles = join(fnDir, "input_classes.stk")
TsOrig = self.inputSet.get().getSamplingRate()
TsRefVol = -1
if self.thereisRefVolume:
TsRefVol = self.refVolume.get().getSamplingRate()
if self.useMaxRes:
self.TsCurrent = max([TsOrig, self.maxResolution.get(), TsRefVol])
self.TsCurrent = self.TsCurrent / 3
Xdim = self.inputSet.get().getDimensions()[0]
self.newXdim = int(round(Xdim * TsOrig / self.TsCurrent))
if self.newXdim < 40:
self.newXdim = int(40)
self.TsCurrent = float(TsOrig) * (float(Xdim) /
float(self.newXdim))
if self.newXdim != Xdim:
self.runJob("xmipp_image_resize",
"-i %s -o %s --fourier %d" %
(self.imgsFn, fnNewParticles, self.newXdim),
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
else:
self.runJob("xmipp_image_convert",
"-i %s -o %s "
"--save_metadata_stack %s" %
(self.imgsFn, fnNewParticles,
join(fnDir, "input_classes.xmd")),
numberOfMpi=1)
# To resample the refVolume if exists with the newXdim calculated
# previously
if self.thereisRefVolume:
fnFilVol = self._getExtraPath('filteredVolume.vol')
self.runJob("xmipp_image_convert",
"-i %s -o %s -t vol" %
(self.refVolume.get().getFileName(), fnFilVol),
numberOfMpi=1)
# TsVol = self.refVolume.get().getSamplingRate()
if self.useMaxRes:
if self.newXdim != Xdim:
self.runJob('xmipp_image_resize',
"-i %s --fourier %d" %
(fnFilVol, self.newXdim),
numberOfMpi=1)
self.runJob('xmipp_transform_window',
"-i %s --size %d" % (fnFilVol, self.newXdim),
numberOfMpi=1)
args = "-i %s --fourier low_pass %f --sampling %f " % (
fnFilVol, self.maxResolution.get(), self.TsCurrent)
self.runJob("xmipp_transform_filter", args, numberOfMpi=1)
if not self.useMaxRes:
inputVolume = self.refVolume.get()
else:
inputVolume = Volume(fnFilVol)
inputVolume.setSamplingRate(self.TsCurrent)
inputVolume.setObjId(self.refVolume.get().getObjId())
fnVolumes = self._getExtraPath('input_volumes.xmd')
row = metadata.Row()
volumeToRow(inputVolume, row, alignType=ALIGN_NONE)
md = emlib.MetaData()
row.writeToMd(md, md.addObject())
md.write(fnVolumes)
def _processMovie(self, movie):
movId = movie.getObjId()
x, y, n = movie.getDim()
iniFrame, lastFrame, _ = movie.getFramesRange()
frame0, frameN = self._getRange(movie)
boxSize = self.boxSize.get()
if movie.hasAlignment() and self.applyAlignment:
shiftX, shiftY = movie.getAlignment().getShifts() # lists.
else:
shiftX = [0] * (lastFrame - iniFrame + 1)
shiftY = shiftX
stkIndex = 0
movieStk = self._getMovieName(movie, '.stk')
movieMdFile = self._getMovieName(movie, '.xmd')
movieMd = md.MetaData()
frameMd = md.MetaData()
frameMdImages = md.MetaData()
frameRow = md.Row()
if self._hasCoordinates(movie):
imgh = ImageHandler()
for frame in range(frame0, frameN + 1):
indx = frame - iniFrame
frameName = self._getFnRelated('frameMic', movId, frame)
frameMdFile = self._getFnRelated('frameMdFile', movId, frame)
coordinatesName = self._getFnRelated('frameCoords', movId,
frame)
frameImages = self._getFnRelated('frameImages', movId, frame)
frameStk = self._getFnRelated('frameStk', movId, frame)
self._writeXmippPosFile(movie, coordinatesName, shiftX[indx],
shiftY[indx])
self.info("Writing frame: %s" % frameName)
# TODO: there is no need to write the frame and then operate
# the input of the first operation should be the movie
movieName = imgh.fixXmippVolumeFileName(movie)
imgh.convert((frame, movieName), frameName)
if self.doRemoveDust:
self.info("Removing Dust")
self._runNoDust(frameName)
self.info("Extracting particles")
args = '-i %(frameName)s --pos %(coordinatesName)s ' \
'-o %(frameImages)s --Xdim %(boxSize)d' % locals()
if self.doInvert:
args += " --invert"
if self.doBorders:
args += " --fillBorders"
args += " --downsampling %f " % self.getBoxScale()
self.runJob('xmipp_micrograph_scissor', args)
cleanPath(frameName)
self.info("Combining particles into one stack.")
frameMdImages.read(frameMdFile)
frameMd.read('particles@%s' % coordinatesName)
frameMd.merge(frameMdImages)
for objId in frameMd:
stkIndex += 1
frameRow.readFromMd(frameMd, objId)
location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
newLocation = (stkIndex, movieStk)
imgh.convert(location, newLocation)
# Fix the name to be accessible from the Project directory
# so we know that the movie stack file will be moved
# to final particles folder
newImageName = '%d@%s' % newLocation
frameRow.setValue(md.MDL_IMAGE, newImageName)
frameRow.setValue(md.MDL_MICROGRAPH_ID, int(movId))
frameRow.setValue(md.MDL_MICROGRAPH, str(movId))
frameRow.setValue(md.MDL_FRAME_ID, int(frame))
frameRow.setValue(md.MDL_PARTICLE_ID,
frameRow.getValue(md.MDL_ITEM_ID))
frameRow.writeToMd(movieMd, movieMd.addObject())
movieMd.addItemId()
movieMd.write(movieMdFile)
cleanPath(frameStk)
if self.doNormalize:
numberOfFrames = frameN - frame0 + 1
self._runNormalize(movieStk, numberOfFrames)
def classifyOneGroup(self, projNumber, projMdBlock, projRef, mdClasses,
mdImages):
""" Classify one of the neighbourhood groups if not empty.
Class information will be stored in output metadata: mdOut
"""
blockSize = md.getSize(projMdBlock)
fnToUse = projMdBlock
if self.maxCLimgs > 0 and blockSize > self.maxCLimgs:
fnToUSe = self._getTmpPath("coneImages.xmd")
self.runJob("xmipp_metadata_utilities","-i %s -o %s --operate random_subset %d"\
%(projMdBlock,fnToUSe,self.maxCLimgs),numberOfMpi=1)
Nclasses = self.directionalClasses.get()
Nlevels = int(math.ceil(math.log(Nclasses) / math.log(2)))
# Skip projection directions with not enough images to
# create a given number of classes
if blockSize / Nclasses < 10:
return
if self.useGpu.get():
count = 0
GpuListCuda = ''
if self.useQueueForSteps() or self.useQueue():
GpuList = os.environ["CUDA_VISIBLE_DEVICES"]
GpuList = GpuList.split(",")
for elem in GpuList:
GpuListCuda = GpuListCuda + str(count) + ' '
count += 1
else:
GpuList = ' '.join([str(elem) for elem in self.getGpuList()])
GpuListAux = ''
for elem in self.getGpuList():
GpuListCuda = GpuListCuda + str(count) + ' '
GpuListAux = GpuListAux + str(elem) + ','
count += 1
os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux
fnDir = self._getExtraPath("direction_%s" % projNumber)
if not exists(fnDir):
makePath(fnDir)
for i in range(self.class2dIterations.get()):
mdRefName = join(fnDir, 'reference.xmd')
if i == 0:
mdRef = md.MetaData()
for j in range(2):
row = md.Row()
row.setValue(md.MDL_REF, j + 1)
row.setValue(md.MDL_IMAGE, projRef)
row.addToMd(mdRef)
mdRef.write(mdRefName, emlib.MD_APPEND)
else:
mdRefName = join(fnDir, "level_%02d" % (i - 1),
"class_classes.xmd")
if not exists(join(fnDir, "level_%02d" % i)):
makePath(join(fnDir, "level_%02d" % i))
args = '-i %s -r %s -o images.xmd --odir %s' \
' --keepBestN 1 --oUpdatedRefs %s ' % (fnToUse, mdRefName, join(fnDir,"level_%02d"%i), 'class_classes')
args += ' --dev %s ' % GpuListCuda
self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1)
copy(
join(fnDir, "level_%02d" % (self.class2dIterations.get() - 1),
"images.xmd"), join(fnDir, "images.xmd"))
# After classification the stk and xmd files should be produced
classesXmd = join(
fnDir, "level_%02d/class_classes.xmd" %
(self.class2dIterations.get() - 1))
classesStk = join(
fnDir, "level_%02d/class_classes.stk" %
(self.class2dIterations.get() - 1))
else:
fnDir = self._getExtraPath("direction_%s" % projNumber)
if not exists(join(fnDir, "level_00")):
makePath(fnDir)
# Run CL2D classification for the images assigned to one direction
args = "-i %s " % fnToUse
args += "--odir %s " % fnDir
args += "--ref0 %s --iter %d --nref %d " % \
(projRef, self.class2dIterations, Nclasses)
args += "--distance correlation --classicalMultiref "
args += "--maxShift %f " % self.maxShift
try:
self.runJob("xmipp_classify_CL2D",
args,
numberOfMpi=self.numberOfMpi.get() *
self.numberOfThreads.get())
except:
return
# After CL2D the stk and xmd files should be produced
classesXmd = join(fnDir, "level_%02d/class_classes.xmd" % Nlevels)
classesStk = join(fnDir, "level_%02d/class_classes.stk" % Nlevels)
# Let's check that the output was produced
if not exists(classesStk):
return
# Run align of the class average and the projection representative
fnAlignRoot = join(fnDir, "classes")
args = "-i %s " % classesStk
args += "--ref %s " % projRef
args += " --oroot %s --iter 1" % fnAlignRoot
self.runJob("xmipp_image_align", args, numberOfMpi=1)
# Apply alignment
args = "-i %s_alignment.xmd --apply_transform" % fnAlignRoot
self.runJob("xmipp_transform_geometry", args, numberOfMpi=1)
for classNo in range(1, Nclasses + 1):
localImagesMd = emlib.MetaData("class%06d_images@%s" %
(classNo, classesXmd))
# New class detected
self.classCount += 1
# Check which images have not been assigned yet to any class
# and assign them to this new class
for objId in localImagesMd:
imgId = localImagesMd.getValue(emlib.MDL_ITEM_ID, objId)
# Add images not classify yet and store their class number
if imgId not in self.classImages:
self.classImages.add(imgId)
newObjId = mdImages.addObject()
mdImages.setValue(emlib.MDL_ITEM_ID, imgId, newObjId)
mdImages.setValue(emlib.MDL_REF2, self.classCount,
newObjId)
newClassId = mdClasses.addObject()
mdClasses.setValue(emlib.MDL_REF, projNumber, newClassId)
mdClasses.setValue(emlib.MDL_REF2, self.classCount, newClassId)
mdClasses.setValue(emlib.MDL_IMAGE,
"%d@%s" % (classNo, classesStk), newClassId)
mdClasses.setValue(emlib.MDL_IMAGE1, projRef, newClassId)
mdClasses.setValue(emlib.MDL_CLASS_COUNT, localImagesMd.size(),
newClassId)
def test_forward_backwards(self):
"""convert transformation matrix to xmipp and back"""
mList = [
[
[0.71461016, 0.63371837, -0.29619813, 1.], # a1
[-0.61309201, 0.77128059, 0.17101008, 2.],
[0.33682409, 0.059391174, 0.93969262, 3.],
[0, 0, 0, 1.]
],
[
[0., 0., -1., 0.], # a2
[0., 1., 0., 0.],
[1., 0., 0., 0.],
[0., 0., 0., 1.]
],
[
[0., 1., 0., 0.], # a3
[0., 0., 1., 0.],
[1., 0., 0., 0.],
[0., 0., 0., 1.]
],
[
[0.22612257, 0.82379508, -0.51983678, 0.], # a4
[-0.88564873, 0.39606407, 0.24240388, 0.],
[0.40557978, 0.40557978, 0.81915206, 0.],
[0., 0., 0., 1.]
],
[
[-0.78850311, -0.24329656, -0.56486255, 0.], # a5
[0.22753462, -0.96866286, 0.099600501, 0.],
[-0.57139379, -0.049990479, 0.81915206, 0.],
[0., 0., 0., 1.]
],
[
[1.0, 0.0, 0.0, 0.0], # a6
[0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0],
[0.0, 0.0, 0.0, 1.0]
],
[
[0., 0., -1., 0.], # a7
[-1., 0., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 0., 1.]
]
]
aList = [np.array(m) for m in mList]
rowa = md.Row()
rowb = md.Row()
labelList = [
md.RLN_ORIENT_ROT, md.RLN_ORIENT_TILT, md.RLN_ORIENT_PSI,
md.RLN_ORIENT_ORIGIN_X, md.RLN_ORIENT_ORIGIN_Y,
md.RLN_ORIENT_ORIGIN_Z
]
for i, a in enumerate(aList):
a = Transform(aList[i])
convert.alignmentToRow(a, rowa, ALIGN_PROJ)
b = convert.rowToAlignment(rowa, ALIGN_PROJ)
convert.alignmentToRow(b, rowb, ALIGN_PROJ)
# same two matrices
self.assertTrue(
np.allclose(a.getMatrix(), b.getMatrix(), rtol=1e-2))
for label in labelList:
auxBtilt = rowb.getValue(label)
auxAtilt = rowa.getValue(label)
# same two rows
self.assertAlmostEqual(auxBtilt, auxAtilt, places=3)
b = convert.rowToAlignment(rowa, ALIGN_PROJ)
convert.alignmentToRow(b, rowb, ALIGN_PROJ)
aMatrix = a.getMatrix()
# aMatrix[0,:] *= -1; aMatrix[2,:] *= -1;
# same two matrices with flip
print("\naMatrix: \n", aMatrix, "\nbMatrix: \n", b.getMatrix())
self.assertTrue(np.allclose(aMatrix, b.getMatrix(), rtol=1e-2))
def runMapBack(self, classId):
for tomo in self.inputTomograms.get().iterItems():
if self.selection == 0:
TsSubtomo = self.inputClasses.get().getSamplingRate()
else:
TsSubtomo = self.inputRef.get().getSamplingRate()
TsTomo = tomo.getSamplingRate()
scaleFactor = TsSubtomo / TsTomo
mdGeometry = lib.MetaData()
if self.selection == 0:
inputSet = self.inputClasses.get().getFirstItem()
ref = self._getExtraPath("reference%d.mrc" % classId)
else:
inputSet = self.inputSubtomos.get()
ref = self.inputRef.get().getFileName()
for subtomo in inputSet.iterItems():
fn = subtomo.getFileName()
if fn.endswith('.mrc'):
fn += ':mrc'
if subtomo.getCoordinate3D().getVolId() == tomo.getObjId() \
or basename(subtomo.getVolName()) == tomo.getBaseName().partition('import_')[0]:
nRow = md.Row()
nRow.setValue(lib.MDL_ITEM_ID, int(subtomo.getObjId()))
coord = subtomo.getCoordinate3D()
coord.setVolume(tomo)
nRow.setValue(
lib.MDL_XCOOR,
int(
coord.getX(const.BOTTOM_LEFT_CORNER) *
scaleFactor))
nRow.setValue(
lib.MDL_YCOOR,
int(
coord.getY(const.BOTTOM_LEFT_CORNER) *
scaleFactor))
nRow.setValue(
lib.MDL_ZCOOR,
int(
coord.getZ(const.BOTTOM_LEFT_CORNER) *
scaleFactor))
# Compute inverse matrix
A = subtomo.getTransform().getMatrix()
subtomo.getTransform().setMatrix(np.linalg.inv(A))
# Convert transform matrix to Euler Angles (rot, tilt, psi)
alignmentToRow(subtomo.getTransform(), nRow, ALIGN_3D)
nRow.addToMd(mdGeometry)
fnGeometry = self._getExtraPath("geometry%d.xmd" % classId)
mdGeometry.write(fnGeometry)
if TsSubtomo != TsTomo:
factor = TsSubtomo / TsTomo
args = "-i %s -o %s --scale %d" % (ref, ref, factor)
self.runJob('xmipp_transform_geometry', args)
if self.paintingType.get() == 0:
painting = 'copy'
elif self.paintingType.get() == 1:
painting = 'avg %d' % self.threshold.get()
elif self.paintingType.get() == 2:
painting = 'highlight %d' % self.constant.get()
elif self.paintingType.get() == 3:
painting = 'copy_binary %f' % self.threshold.get()
tomogram = self._getExtraPath("tomogram_%d.mrc" % tomo.getObjId())
args = " -i %s -o %s --geom %s --ref %s --method %s" % (
tomogram, tomogram,
self._getExtraPath("geometry%d.xmd" % classId), ref, painting)
self.runJob("xmipp_tomo_map_back", args)
