def _testInv(matrix): matrix = np.array(matrix) a = Transform(matrix) row1 = md.Row() convert.alignmentToRow(a, row1, alignType=ALIGN_2D) row2 = None row3 = md.Row() convert.alignmentToRow(a, row3, alignType=ALIGN_PROJ) return row1, row2, row3
def _testInv(matrix): matrix = np.array(matrix) a = Transform(matrix) row = metadata.Row() alignmentToRow(a, row, alignType=ALIGN_2D) row2 = metadata.Row() alignmentToRow(a, row2, alignType=ALIGN_3D) row3 = metadata.Row() alignmentToRow(a, row3, alignType=ALIGN_PROJ) return row, row2, row3
def alignParticlesStep(self): fhInputTranMat = self._getExtraPath('transformation-matrix.txt') outParticlesFn = self._getExtraPath('outputParticles.xmd') transMatFromFile = np.loadtxt(fhInputTranMat) transformationMat = np.reshape(transMatFromFile, (4, 4)) transform = Transform() transform.setMatrix(transformationMat) resultMat = Transform() outputParts = md.MetaData() mdToAlign = md.MetaData(self.imgsInputFn) for row in md.iterRows(mdToAlign): inMat = rowToAlignment(row, ALIGN_PROJ) partTransformMat = inMat.getMatrix() partTransformMatrix = np.matrix(partTransformMat) newTransformMatrix = np.matmul(transformationMat, partTransformMatrix) resultMat.setMatrix(newTransformMatrix) rowOut = md.Row() rowOut.copyFromRow(row) alignmentToRow(resultMat, rowOut, ALIGN_PROJ) rowOut.addToMd(outputParts) outputParts.write(outParticlesFn) cleanPath(self.imgsInputFn)
def createAngDistributionSqlite(self, sqliteFn, numberOfParticles, itemDataIterator): if not os.path.exists(sqliteFn): # List of list of 3 elements containing angleTilt, anglePsi, weight projectionList = [] def getCloseProjection(angleRot, angleTilt): """ Get an existing projection close to angleRot, angleTilt. Return None if not found close enough. """ for projection in projectionList: if (abs(projection[0] - angleRot) <= 0.5 and abs(projection[1] - angleTilt) <= 0.5): return projection return None weight = 1. / numberOfParticles for angleRot, angleTilt in itemDataIterator: projection = getCloseProjection(angleRot, angleTilt) if projection is None: projectionList.append([angleRot, angleTilt, weight]) else: projection[2] = projection[2] + weight mdProj = md.MetaData() for projection in projectionList: mdRow = md.Row() mdRow.setValue(md.MDL_ANGLE_ROT, projection[0]) mdRow.setValue(md.MDL_ANGLE_TILT, projection[1]) mdRow.setValue(md.MDL_WEIGHT, projection[2]) mdRow.writeToMd(mdProj, mdProj.addObject()) mdProj.write(sqliteFn)
def mergeVolumesStep(self, numOfRuns): mdOut = md.MetaData() volFnList = [] clsDistList = [] accList = [] for run in range(1, numOfRuns + 1): it = self._lastIter(run) modelFile = self._getFileName('model', ruNum=run, iter=it) mdIn = md.MetaData('model_classes@%s' % modelFile) for row in md.iterRows(mdIn, md.RLN_MLMODEL_REF_IMAGE): volFn = row.getValue(md.RLN_MLMODEL_REF_IMAGE) clsDist = row.getValue('rlnClassDistribution') accRot = row.getValue('rlnAccuracyRotations') if accRot <= 90: volFnList.append(volFn) clsDistList.append(clsDist) accList.append(accRot) self.std = np.std(accList) score = self._estimateScore(accList, clsDistList) tupleList = [(fn, s) for fn, s in zip(volFnList, score)] nVols = self.numOfVols.get() sortList = sorted(tupleList, reverse=True, key=lambda x: x[1])[0:nVols] row = md.Row() for val in sortList: fn, score = val row.setValue(md.RLN_MLMODEL_REF_IMAGE, fn) row.addToMd(mdOut) mdOut.write(self._getRefStar())
def correlationCudaStep(self, thIdx, gpuId, totalGpu): mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc)) self.numCones = mdNumCones.size() # Cuda Correlation step - creating the metadata predCones = np.loadtxt(self._getConePrediction()) mdConeList = [] numMax = int(self.numConesSelected) for i in range(self.numCones): mdConeList.append(emlib.MetaData()) mdIn = emlib.MetaData(self.imgsFn) for i in range(self.numCones): idx = i + 1 if (idx % totalGpu) != thIdx: continue #modelFn = 'modelCone%d_aux' % idx #f = open(join(self._getExtraPath(), modelFn+'.txt'),'r') #mae = float(f.readline()) #f.close() modelFn = 'modelCone%d' % idx positions = [] for n in range(numMax): posAux = np.where(predCones[:, (n * 2) + 1] == (i + 1)) positions = positions + (np.ndarray.tolist(posAux[0])) if len(positions) > 0 and exists(self._getExtraPath(modelFn + '.h5')): print("Classifying cone ", idx, "in GPU ", gpuId) for pos in positions: id = pos + 1 row = md.Row() row.readFromMd(mdIn, id) row.addToMd(mdConeList[i]) fnExpCone = self._getExtraPath('metadataCone%d.xmd' % (i + 1)) mdConeList[i].write(fnExpCone) fnProjCone = self._getExtraPath('projectionsCudaCorr%d.xmd' % (i + 1)) self.runJob("xmipp_metadata_utilities", "-i %s --fill ref lineal 1 1 " % (fnProjCone), numberOfMpi=1) fnOutCone = self._getOutCone(i + 1) if not exists(self._getExtraPath(fnOutCone)): params = ' -i %s' % fnExpCone params += ' -r %s' % fnProjCone params += ' -o %s' % self._getExtraPath(fnOutCone) params += ' --dev %s '%(gpuId) self.runJob("xmipp_cuda_align_significant", params, numberOfMpi=1)
def _convertRef(self): ih = ImageHandler() inputObj = self.inputVolumes.get() row = md.Row() refMd = md.MetaData() for vol in inputObj: newVolFn = self._convertVol(ih, vol) row.setValue(md.RLN_MLMODEL_REF_IMAGE, newVolFn) row.addToMd(refMd) refMd.write(self._getRefStar())
def test_rowToImage(self): row = metadata.Row() index = 1 filename = 'images.stk' row.setValue(emlib.MDL_ITEM_ID, 1) row.setValue(emlib.MDL_IMAGE, '%d@%s' % (index, filename)) img = rowToImage(row, emlib.MDL_IMAGE, Particle) # Check correct index and filename self.assertEquals(index, img.getIndex()) self.assertEquals(filename, img.getFileName())
def createRefMd(self, vols): refVols = self._getExtraPath('ref_volumes.xmd') mdFn = md.MetaData() if self.isSetOfVolumes(): for vol in vols: imgId = vol.getObjId() row = md.Row() row.setValue(md.MDL_ITEM_ID, int(imgId)) row.setValue(md.MDL_IMAGE, getImageLocation(vol)) row.setValue(md.MDL_ENABLED, 1) row.addToMd(mdFn) else: imgId = vols.getObjId() row = md.Row() row.setValue(md.MDL_ITEM_ID, int(imgId)) row.setValue(md.MDL_IMAGE, getImageLocation(vols)) row.setValue(md.MDL_ENABLED, 1) row.addToMd(mdFn) mdFn.write(refVols, md.MD_APPEND)
def createInputMd(self, vols): fnVols = self._getExtraPath('input_volumes.xmd') if self.copyAlignment: alignType = vols.getAlignment() else: alignType = ALIGN_NONE writeSetOfVolumes(vols, fnVols, postprocessImageRow=self._postprocessVolumeRow, alignType=alignType) if not vols.hasAlignment() or not self.copyAlignment: mdFn = md.MetaData(fnVols) mdFn.fillConstant(md.MDL_ANGLE_ROT, 0.) mdFn.fillConstant(md.MDL_ANGLE_TILT, 0.) mdFn.fillConstant(md.MDL_ANGLE_PSI, 0.) mdFn.fillConstant(md.MDL_SHIFT_X, 0.) mdFn.fillConstant(md.MDL_SHIFT_Y, 0.) mdFn.fillConstant(md.MDL_SHIFT_Z, 0.) mdFn.write(fnVols, md.MD_OVERWRITE) # set missing angles missType = ['wedge_y', 'wedge_x', 'pyramid', 'cone'] missNum = self.missingDataType.get() missAng = self.missingAng.get() missDataFn = self._getExtraPath('wedges.xmd') missAngValues = str(missAng).strip().split() thetaY0, thetaYF, thetaX0, thetaXF = 0, 0, 0, 0 mdFn = md.MetaData() if missNum == MISSING_WEDGE_X: thetaX0, thetaXF = missAngValues elif missNum == MISSING_WEDGE_Y: thetaY0, thetaYF = missAngValues elif missNum == MISSING_PYRAMID: thetaY0, thetaYF, thetaX0, thetaXF = missAngValues else: # MISSING_CONE thetaY0 = missAngValues[0] for i in range(1, vols.getSize() + 1): row = md.Row() row.setValue(md.MDL_MISSINGREGION_NR, missNum + 1) row.setValue(md.MDL_MISSINGREGION_TYPE, missType[missNum]) row.setValue(md.MDL_MISSINGREGION_THX0, float(thetaX0)) row.setValue(md.MDL_MISSINGREGION_THXF, float(thetaXF)) row.setValue(md.MDL_MISSINGREGION_THY0, float(thetaY0)) row.setValue(md.MDL_MISSINGREGION_THYF, float(thetaYF)) row.addToMd(mdFn) mdFn.write(missDataFn, md.MD_APPEND)
def test_writeSetOfDefocusGroups(self): #TODO: FIX THIS test according to the new SetOfDefocusGroup return #reference metadata md = emlib.MetaData() objId = md.addObject() defocusGroupRow = metadata.Row() defocusGroupRow.setValue(emlib.MDL_ENABLED, 1) defocusGroupRow.setValue(emlib.MDL_CTF_GROUP, 1) defocusGroupRow.setValue(emlib.MDL_MIN, 2000.) defocusGroupRow.setValue(emlib.MDL_MAX, 2500.) defocusGroupRow.setValue(emlib.MDL_AVG, 2100.) defocusGroupRow.writeToMd(md, objId) objId = md.addObject() defocusGroupRow.setValue(emlib.MDL_ENABLED, 1) defocusGroupRow.setValue(emlib.MDL_CTF_GROUP, 2) defocusGroupRow.setValue(emlib.MDL_MIN, 3000.) defocusGroupRow.setValue(emlib.MDL_MAX, 5500.) defocusGroupRow.setValue(emlib.MDL_AVG, 5000.) defocusGroupRow.writeToMd(md, objId) # fnScipion = self.getOutputPath("writeSetOfDefocusGroups.sqlite") setOfDefocus = SetOfDefocusGroup(filename=fnScipion) df = DefocusGroup() df.setDefocusMin(2000.) df.setDefocusMax(2500.) df.setDefocusAvg(2100.) setOfDefocus.append(df) df.cleanObjId() df.setDefocusMin(3000) df.setDefocusMax(5500) df.setDefocusAvg(5000) setOfDefocus.append(df) fnXmipp = self.getOutputPath("writeSetOfDefocusGroups.xmd") fnScipion = self.getOutputPath("writeSetOfDefocusGroups2.xmd") writeSetOfDefocusGroups(setOfDefocus, fnXmipp) mdAux = emlib.MetaData(fnXmipp) md.write(fnScipion) print("Comparing metadatas: \n%s\n%s" % (fnXmipp, fnScipion)) self.assertEqual(md, mdAux, "test writeSetOfDefocusGroups fails")
def test_rowToCtfModel(self): row = metadata.Row() row.setValue(emlib.MDL_CTF_DEFOCUSU, 2520.) row.setValue(emlib.MDL_CTF_DEFOCUSV, 2510.) row.setValue(emlib.MDL_CTF_DEFOCUS_ANGLE, 45.) ctf = rowToCtfModel(row) # Check that the ctf object was properly set self.assertTrue(ctf.equalAttributes(self.getCTF(2520., 2510., 45.))) # Check when the EMX standarization takes place row.setValue(emlib.MDL_CTF_DEFOCUSV, 2530.) ctf = rowToCtfModel(row) self.assertTrue(ctf.equalAttributes(self.getCTF(2530., 2520., 135.))) # When one of CTF labels is missing, None should be returned row.removeLabel(emlib.MDL_CTF_DEFOCUSV) ctf = rowToCtfModel(row) self.assertIsNone(ctf)
def _convertRef(self): ih = ImageHandler() inputObj = self.inputVolumes.get() subset = SetOfVolumes(filename=":memory:") refMd = md.MetaData() for vol in inputObj.iterItems(orderBy='RANDOM()'): subset.append(vol) subsetSize = self.numOfVols.get() minSize = min(subsetSize, inputObj.getSize()) if subset.getSize() <= minSize: row = md.Row() newVolFn = self._convertVol(ih, vol) row.setValue(md.RLN_MLMODEL_REF_IMAGE, newVolFn) row.addToMd(refMd) else: break refMd.write(self._getRefStar())
def _writeXmippPosFile(self, movie, coordinatesName, shiftX, shiftY): """ Create Xmipp coordinate files to be extracted from the frames of the movie. """ coordSet = self.getCoords() mData = md.MetaData() coordRow = md.Row() for coord in coordSet.iterCoordinates(movie.getObjId()): coord.shiftX(int(round(float(shiftX)))) coord.shiftY(int(round(float(shiftY)))) coordinateToRow(coord, coordRow) coordRow.writeToMd(mData, mData.addObject()) self.info("Writing coordinates metadata: %s, with shifts: %s %s" % (coordinatesName, shiftX, shiftY)) mData.write('particles@' + coordinatesName)
def generateOutputClasses(self, classesOut, firstTime): if firstTime: self._saveFileDataClasses(classesOut, self._getExtraPath('last_classes.xmd')) # Add the two new classes for i in range(1, 3): mdImgsInClass = md.MetaData('class%06d_images@%s' % (i, classesOut)) mdImgsInClass.write(self._getExtraPath('dataClass%06d.xmd' % i)) return finalMetadata = self._getExtraPath('aux_classes.stk') lastMetadata = self._getExtraPath('last_classes.xmd') newMetadata = classesOut total = self.averageClasses(finalMetadata, lastMetadata, newMetadata, False) copy(self._getExtraPath('aux_classes.stk'), self._getExtraPath('last_classes.stk')) mdAll = md.MetaData() newRef = 1 for i in total: if i != 0: row = md.Row() row.setValue(md.MDL_REF, newRef) row.setValue(md.MDL_IMAGE, '%06d@' % newRef + finalMetadata) row.setValue(md.MDL_CLASS_COUNT, i) row.addToMd(mdAll) newRef += 1 mdAll.write('classes@' + finalMetadata[:-3] + 'xmd', MD_APPEND) copy(self._getExtraPath('aux_classes.xmd'), self._getExtraPath('last_classes.xmd')) newRef = 1 for i, val in enumerate(total): if val != 0: self._unionDataClass(classesOut, i + 1, newRef) newRef += 1
def correlationSignificantStep(self): mdNumCones = emlib.MetaData(self._getExtraPath(self._fnConeCenterDoc)) self.numCones = mdNumCones.size() # Cuda Correlation step - creating the metadata predCones = np.loadtxt(self._getConePrediction()) mdConeList = [] numMax = int(self.numConesSelected) for i in range(self.numCones): mdConeList.append(emlib.MetaData()) mdIn = emlib.MetaData(self.imgsFn) for i in range(self.numCones): print("Classifying cone ", i + 1) positions = [] for n in range(numMax): posAux = np.where(predCones[:, (n * 2) + 1] == (i + 1)) positions = positions + (np.ndarray.tolist(posAux[0])) if len(positions) > 0: for pos in positions: id = pos + 1 row = md.Row() row.readFromMd(mdIn, id) row.addToMd(mdConeList[i]) fnExpCone = self._getExtraPath('metadataCone%d.xmd' % (i + 1)) mdConeList[i].write(fnExpCone) fnProjCone = self._getExtraPath('projectionsCudaCorr%d.xmd' % (i + 1)) fnOutCone = self._getOutCone(i + 1) if not exists(self._getExtraPath(fnOutCone)): # Correlation step - calling significant program args = '-i %s --initgallery %s --odir %s --dontReconstruct --useForValidation %d ' \ '--dontCheckMirrors --maxShift 30' % (fnExpCone, fnProjCone, self._getExtraPath(), 1) self.runJob('xmipp_reconstruct_significant', args, numberOfMpi=self.myMPI.get()) copy(self._getExtraPath('images_significant_iter001_00.xmd'), self._getExtraPath(fnOutCone)) remove(self._getExtraPath('angles_iter001_00.xmd')) remove(self._getExtraPath('images_significant_iter001_00.xmd'))
def writeSetOfMicrographs(micSet, filename): """ Simplified function borrowed from xmipp. """ mdata = md.MetaData() for img in micSet: objId = mdata.addObject() imgRow = md.Row() imgRow.setValue(md.MDL_ITEM_ID, int(objId)) index, fname = img.getLocation() fn = ImageHandler.locationToXmipp((index, fname)) imgRow.setValue(md.MDL_MICROGRAPH, fn) if img.isEnabled(): enabled = 1 else: enabled = -1 imgRow.setValue(md.MDL_ENABLED, enabled) imgRow.writeToMd(mdata, objId) mdata.write('Micrographs@%s' % filename)
def applyAlignmentStep(self, inputFn): inputSt = self.inputSubtomograms.get() # Window subtomograms twice their size windowedStk = self._getExtraPath('windowed_subtomograms.stk') self.runJob( 'xmipp_transform_window', '-i %s -o %s --size %d --save_metadata_stack' % (inputFn, windowedStk, 2 * inputSt.getFirstItem().getDim()[0]), numberOfMpi=1) # Add input transform matrix to md generated by xmipp_transform_window mdWindow = md.MetaData(self._getExtraPath('windowed_subtomograms.xmd')) mdWindowTransform = md.MetaData() idList = list(inputSt.getIdSet()) for row in md.iterRows(mdWindow): rowOut = md.Row() rowOut.copyFromRow(row) id = row.getValue(MDL_IMAGE) id = id.split('@')[0] id = id.strip('0') alignmentToRow(inputSt[(idList[int(id) - 1])].getTransform(), rowOut, pwem.ALIGN_3D) rowOut.addToMd(mdWindowTransform) mdWindowTransform.write( self._getExtraPath("window_with_original_geometry.xmd")) # Align subtomograms self.runJob( 'xmipp_transform_geometry', '-i %s -o %s --apply_transform' % (self._getExtraPath("window_with_original_geometry.xmd"), self._getExtraPath('aligned_subtomograms.stk'))) # Window subtomograms to their original size alignStk = self._getExtraPath('aligned_subtomograms.stk') outputStk = self._getPath('output_subtomograms.stk') self.runJob('xmipp_transform_window', '-i %s -o %s --size %d ' % (alignStk, outputStk, self.inputSubtomograms.get().getFirstItem().getDim()[0]), numberOfMpi=1) return [outputStk]
def _writeMDFinal(self, allInFns, coneFns, coneCCs, mdCones, shiftX, shiftY, fnFinal): mdFinal = emlib.MetaData() row = md.Row() for myFn in allInFns: myCCs = [] myCones = [] myPos = [] for n in range(self.numCones): if myFn in coneFns[n]: pos = coneFns[n].index(myFn) myPos.append(pos) if abs(shiftX[n][pos])<30 and abs(shiftY[n][pos])<30: myCCs.append(coneCCs[n][pos]) else: myCCs.append(0) myCones.append(n + 1) if len(myPos) > 0: if max(myCCs)==0: continue coneMax = myCones[myCCs.index(max(myCCs))] objId = myPos[myCCs.index(max(myCCs))] + 1 row.readFromMd(mdCones[coneMax - 1], objId) row.addToMd(mdFinal) mdFinal.write(fnFinal)
def realignStep(self): inputMdName = self._getExtraPath('inputClasses.xmd') writeSetOfClasses2D(self.inputClasses.get(), inputMdName, writeParticles=True) centeredStackName = self._getExtraPath('centeredStack.stk') self._params = {'input': inputMdName, 'output': centeredStackName} args = ('-i %(input)s -o %(output)s --save_metadata_transform') self.runJob("xmipp_transform_center_image", args % self._params, numberOfMpi=1) centeredMdName = centeredStackName.replace('stk', 'xmd') centeredMd = md.MetaData(centeredMdName) centeredStack = md.MetaData(centeredStackName) listName = [] listTransform=[] for rowStk in md.iterRows(centeredStack): listName.append(rowStk.getValue(md.MDL_IMAGE)) for rowMd in md.iterRows(centeredMd): listTransform.append(rowToAlignment(rowMd, ALIGN_2D)) mdNewClasses = md.MetaData() for i, row in enumerate(md.iterRows(inputMdName)): newRow = md.Row() newRow.setValue(md.MDL_IMAGE, listName[i]) refNum = row.getValue(md.MDL_REF) newRow.setValue(md.MDL_REF, refNum) classCount = row.getValue(md.MDL_CLASS_COUNT) newRow.setValue(md.MDL_CLASS_COUNT, classCount) newRow.addToMd(mdNewClasses) mdNewClasses.write('classes@' + self._getExtraPath('final_classes.xmd'), MD_APPEND) mdImages = md.MetaData() i=0 mdBlocks = md.getBlocksInMetaDataFile(inputMdName) resultMat = Transform() for block in mdBlocks: if block.startswith('class00'): newMat = listTransform[i] newMatrix = newMat.getMatrix() mdClass = md.MetaData(block + "@" + inputMdName) mdNewClass = md.MetaData() i+=1 for rowIn in md.iterRows(mdClass): #To create the transformation matrix (and its parameters) # for the realigned particles if rowIn.getValue(md.MDL_ANGLE_PSI)!=0: flag_psi=True if rowIn.getValue(md.MDL_ANGLE_ROT)!=0: flag_psi=False inMat = rowToAlignment(rowIn, ALIGN_2D) inMatrix = inMat.getMatrix() resultMatrix = np.dot(newMatrix,inMatrix) resultMat.setMatrix(resultMatrix) rowOut=md.Row() rowOut.copyFromRow(rowIn) alignmentToRow(resultMat, rowOut, ALIGN_2D) if flag_psi==False: newAngle = rowOut.getValue(md.MDL_ANGLE_PSI) rowOut.setValue(md.MDL_ANGLE_PSI, 0.) rowOut.setValue(md.MDL_ANGLE_ROT, newAngle) #To create the new coordinates for the realigned particles inPoint = np.array([[0.],[0.],[0.],[1.]]) invResultMat = np.linalg.inv(resultMatrix) centerPoint = np.dot(invResultMat,inPoint) rowOut.setValue(md.MDL_XCOOR, rowOut.getValue( md.MDL_XCOOR)+int(centerPoint[0])) rowOut.setValue(md.MDL_YCOOR, rowOut.getValue( md.MDL_YCOOR)+int(centerPoint[1])) rowOut.addToMd(mdNewClass) mdNewClass.write(block + "@" + self._getExtraPath( 'final_classes.xmd'), MD_APPEND) mdImages.unionAll(mdNewClass) mdImages.write(self._getExtraPath('final_images.xmd'))
def rowFromMd(mdata, objId): row = md.Row() row.readFromMd(mdata, objId) return row
def test_forward_backwards(self): """convert transformation matrixt to xmipp and back""" mList = [ [ [0.71461016, 0.63371837, -0.29619813, 1.], #a1 [-0.61309201, 0.77128059, 0.17101008, 2.], [0.33682409, 0.059391174, 0.93969262, 3.], [0, 0, 0, 1.] ], [ [0., 0., -1., 0.], #a2 [0., 1., 0., 0.], [1., 0., 0., 0.], [0., 0., 0., 1.] ], [ [0., 1., 0., 0.], #a3 [0., 0., 1., 0.], [1., 0., 0., 0.], [0., 0., 0., 1.] ], [ [0.22612257, 0.82379508, -0.51983678, 0.], #a4 [-0.88564873, 0.39606407, 0.24240388, 0.], [0.40557978, 0.40557978, 0.81915206, 0.], [0., 0., 0., 1.] ], [ [-0.78850311, -0.24329656, -0.56486255, 0.], #a5 [0.22753462, -0.96866286, 0.099600501, 0.], [-0.57139379, -0.049990479, 0.81915206, 0.], [0., 0., 0., 1.] ], [ [1.0, 0.0, 0.0, 0.0], #a6 [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0] ], [ [0., 0., -1., 0.], #a7 [-1., 0., 0., 0.], [0., 1., 0., 0.], [0., 0., 0., 1.] ] ] aList = [np.array(m) for m in mList] rowa = metadata.Row() rowap = metadata.Row() rowb = metadata.Row() rowb1 = metadata.Row() rowb2 = metadata.Row() rowb3 = metadata.Row() labelList = [ emlib.MDL_ANGLE_ROT, emlib.MDL_ANGLE_TILT, emlib.MDL_ANGLE_PSI, emlib.MDL_SHIFT_X, emlib.MDL_SHIFT_Y, emlib.MDL_SHIFT_Z ] for i, a in enumerate(aList): a = Transform(aList[i]) alignmentToRow(a, rowa, ALIGN_PROJ) b = rowToAlignment(rowa, ALIGN_PROJ) alignmentToRow(b, rowb, ALIGN_PROJ) #same two matrices self.assertTrue( np.allclose(a.getMatrix(), b.getMatrix(), rtol=1e-2)) for label in labelList: auxBtilt = rowb.getValue(label) auxAtilt = rowa.getValue(label) #same two rows self.assertAlmostEqual(auxBtilt, auxAtilt, places=3, msg=None, delta=None) rowa.setValue(emlib.MDL_FLIP, True) aflip = rowToAlignment(rowa, ALIGN_PROJ) alignmentToRow(aflip, rowap, ALIGN_PROJ) aflipp = rowToAlignment(rowap, ALIGN_PROJ) self.assertTrue( np.allclose(aflip.getMatrix(), aflipp.getMatrix(), rtol=1e-2))
def applyAlignStep(self): """Align subtomograms to be in the same orientation as the reference""" inputSet = self.inputSubtomos.get() outputFn = self._getExtraPath('input_subtomos.xmd') if inputSet.getFirstItem().getFileName().endswith('.mrc') or \ inputSet.getFirstItem().getFileName().endswith('.map'): S = self._createSetOfSubTomograms() S.setSamplingRate(inputSet.getSamplingRate()) for subtomo in inputSet: s = subtomo.clone() s.setFileName(subtomo.getFileName() + ':mrc') S.append(s) writeSetOfVolumes(S, outputFn, alignType=ALIGN_3D) else: writeSetOfVolumes(inputSet, outputFn, alignType=ALIGN_3D) # Window subtomograms twice their size windowedStk = self._getExtraPath('windowed_subtomograms.stk') self.runJob( 'xmipp_transform_window', '-i %s -o %s --size %d --save_metadata_stack' % (outputFn, windowedStk, 2 * inputSet.getFirstItem().getDim()[0]), numberOfMpi=1) # Add input transform matrix to md generated by xmipp_transform_window mdWindow = md.MetaData(self._getExtraPath('windowed_subtomograms.xmd')) mdWindowTransform = md.MetaData() idList = list(inputSet.getIdSet()) for row in md.iterRows(mdWindow): rowOut = md.Row() rowOut.copyFromRow(row) id = row.getValue(MDL_IMAGE) id = id.split('@')[0] id = id.strip('0') alignmentToRow(inputSet[(idList[int(id) - 1])].getTransform(), rowOut, ALIGN_3D) rowOut.addToMd(mdWindowTransform) mdWindowTransform.write( self._getExtraPath("window_with_original_geometry.xmd")) # Align subtomograms self.runJob( 'xmipp_transform_geometry', '-i %s -o %s --apply_transform --dont_wrap' % (self._getExtraPath("window_with_original_geometry.xmd"), self._getExtraPath('aligned_subtomograms.stk'))) # Window subtomograms to their original size alignStk = self._getExtraPath('aligned_subtomograms.stk') outputStk = self._getExtraPath('output_subtomograms.stk') self.runJob('xmipp_transform_window', '-i %s -o %s --size %d ' % (alignStk, outputStk, inputSet.getFirstItem().getDim()[0]), numberOfMpi=1) self.alignedSet = self._createSetOfSubTomograms() self.alignedSet.copyInfo(inputSet) inputMd = self._getExtraPath('output_subtomograms.stk') self.alignedSet.copyItems(inputSet, updateItemCallback=self._updateItemAlign, itemDataIterator=md.iterRows( inputMd, sortByLabel=md.MDL_ITEM_ID))
def convertInputStep(self, classesFn): inputSet = self.inputSet.get() if isinstance(inputSet, SetOfClasses2D): writeSetOfClasses2D(inputSet, classesFn, writeParticles=False) else: writeSetOfParticles(inputSet, classesFn) # To re-sample input images fnDir = self._getExtraPath() fnNewParticles = join(fnDir, "input_classes.stk") TsOrig = self.inputSet.get().getSamplingRate() TsRefVol = -1 if self.thereisRefVolume: TsRefVol = self.refVolume.get().getSamplingRate() if self.useMaxRes: self.TsCurrent = max([TsOrig, self.maxResolution.get(), TsRefVol]) self.TsCurrent = self.TsCurrent / 3 Xdim = self.inputSet.get().getDimensions()[0] self.newXdim = int(round(Xdim * TsOrig / self.TsCurrent)) if self.newXdim < 40: self.newXdim = int(40) self.TsCurrent = float(TsOrig) * (float(Xdim) / float(self.newXdim)) if self.newXdim != Xdim: self.runJob("xmipp_image_resize", "-i %s -o %s --fourier %d" % (self.imgsFn, fnNewParticles, self.newXdim), numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get()) else: self.runJob("xmipp_image_convert", "-i %s -o %s " "--save_metadata_stack %s" % (self.imgsFn, fnNewParticles, join(fnDir, "input_classes.xmd")), numberOfMpi=1) # To resample the refVolume if exists with the newXdim calculated # previously if self.thereisRefVolume: fnFilVol = self._getExtraPath('filteredVolume.vol') self.runJob("xmipp_image_convert", "-i %s -o %s -t vol" % (self.refVolume.get().getFileName(), fnFilVol), numberOfMpi=1) # TsVol = self.refVolume.get().getSamplingRate() if self.useMaxRes: if self.newXdim != Xdim: self.runJob('xmipp_image_resize', "-i %s --fourier %d" % (fnFilVol, self.newXdim), numberOfMpi=1) self.runJob('xmipp_transform_window', "-i %s --size %d" % (fnFilVol, self.newXdim), numberOfMpi=1) args = "-i %s --fourier low_pass %f --sampling %f " % ( fnFilVol, self.maxResolution.get(), self.TsCurrent) self.runJob("xmipp_transform_filter", args, numberOfMpi=1) if not self.useMaxRes: inputVolume = self.refVolume.get() else: inputVolume = Volume(fnFilVol) inputVolume.setSamplingRate(self.TsCurrent) inputVolume.setObjId(self.refVolume.get().getObjId()) fnVolumes = self._getExtraPath('input_volumes.xmd') row = metadata.Row() volumeToRow(inputVolume, row, alignType=ALIGN_NONE) md = emlib.MetaData() row.writeToMd(md, md.addObject()) md.write(fnVolumes)
def _processMovie(self, movie): movId = movie.getObjId() x, y, n = movie.getDim() iniFrame, lastFrame, _ = movie.getFramesRange() frame0, frameN = self._getRange(movie) boxSize = self.boxSize.get() if movie.hasAlignment() and self.applyAlignment: shiftX, shiftY = movie.getAlignment().getShifts() # lists. else: shiftX = [0] * (lastFrame - iniFrame + 1) shiftY = shiftX stkIndex = 0 movieStk = self._getMovieName(movie, '.stk') movieMdFile = self._getMovieName(movie, '.xmd') movieMd = md.MetaData() frameMd = md.MetaData() frameMdImages = md.MetaData() frameRow = md.Row() if self._hasCoordinates(movie): imgh = ImageHandler() for frame in range(frame0, frameN + 1): indx = frame - iniFrame frameName = self._getFnRelated('frameMic', movId, frame) frameMdFile = self._getFnRelated('frameMdFile', movId, frame) coordinatesName = self._getFnRelated('frameCoords', movId, frame) frameImages = self._getFnRelated('frameImages', movId, frame) frameStk = self._getFnRelated('frameStk', movId, frame) self._writeXmippPosFile(movie, coordinatesName, shiftX[indx], shiftY[indx]) self.info("Writing frame: %s" % frameName) # TODO: there is no need to write the frame and then operate # the input of the first operation should be the movie movieName = imgh.fixXmippVolumeFileName(movie) imgh.convert((frame, movieName), frameName) if self.doRemoveDust: self.info("Removing Dust") self._runNoDust(frameName) self.info("Extracting particles") args = '-i %(frameName)s --pos %(coordinatesName)s ' \ '-o %(frameImages)s --Xdim %(boxSize)d' % locals() if self.doInvert: args += " --invert" if self.doBorders: args += " --fillBorders" args += " --downsampling %f " % self.getBoxScale() self.runJob('xmipp_micrograph_scissor', args) cleanPath(frameName) self.info("Combining particles into one stack.") frameMdImages.read(frameMdFile) frameMd.read('particles@%s' % coordinatesName) frameMd.merge(frameMdImages) for objId in frameMd: stkIndex += 1 frameRow.readFromMd(frameMd, objId) location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE)) newLocation = (stkIndex, movieStk) imgh.convert(location, newLocation) # Fix the name to be accessible from the Project directory # so we know that the movie stack file will be moved # to final particles folder newImageName = '%d@%s' % newLocation frameRow.setValue(md.MDL_IMAGE, newImageName) frameRow.setValue(md.MDL_MICROGRAPH_ID, int(movId)) frameRow.setValue(md.MDL_MICROGRAPH, str(movId)) frameRow.setValue(md.MDL_FRAME_ID, int(frame)) frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID)) frameRow.writeToMd(movieMd, movieMd.addObject()) movieMd.addItemId() movieMd.write(movieMdFile) cleanPath(frameStk) if self.doNormalize: numberOfFrames = frameN - frame0 + 1 self._runNormalize(movieStk, numberOfFrames)
def classifyOneGroup(self, projNumber, projMdBlock, projRef, mdClasses, mdImages): """ Classify one of the neighbourhood groups if not empty. Class information will be stored in output metadata: mdOut """ blockSize = md.getSize(projMdBlock) fnToUse = projMdBlock if self.maxCLimgs > 0 and blockSize > self.maxCLimgs: fnToUSe = self._getTmpPath("coneImages.xmd") self.runJob("xmipp_metadata_utilities","-i %s -o %s --operate random_subset %d"\ %(projMdBlock,fnToUSe,self.maxCLimgs),numberOfMpi=1) Nclasses = self.directionalClasses.get() Nlevels = int(math.ceil(math.log(Nclasses) / math.log(2))) # Skip projection directions with not enough images to # create a given number of classes if blockSize / Nclasses < 10: return if self.useGpu.get(): count = 0 GpuListCuda = '' if self.useQueueForSteps() or self.useQueue(): GpuList = os.environ["CUDA_VISIBLE_DEVICES"] GpuList = GpuList.split(",") for elem in GpuList: GpuListCuda = GpuListCuda + str(count) + ' ' count += 1 else: GpuList = ' '.join([str(elem) for elem in self.getGpuList()]) GpuListAux = '' for elem in self.getGpuList(): GpuListCuda = GpuListCuda + str(count) + ' ' GpuListAux = GpuListAux + str(elem) + ',' count += 1 os.environ["CUDA_VISIBLE_DEVICES"] = GpuListAux fnDir = self._getExtraPath("direction_%s" % projNumber) if not exists(fnDir): makePath(fnDir) for i in range(self.class2dIterations.get()): mdRefName = join(fnDir, 'reference.xmd') if i == 0: mdRef = md.MetaData() for j in range(2): row = md.Row() row.setValue(md.MDL_REF, j + 1) row.setValue(md.MDL_IMAGE, projRef) row.addToMd(mdRef) mdRef.write(mdRefName, emlib.MD_APPEND) else: mdRefName = join(fnDir, "level_%02d" % (i - 1), "class_classes.xmd") if not exists(join(fnDir, "level_%02d" % i)): makePath(join(fnDir, "level_%02d" % i)) args = '-i %s -r %s -o images.xmd --odir %s' \ ' --keepBestN 1 --oUpdatedRefs %s ' % (fnToUse, mdRefName, join(fnDir,"level_%02d"%i), 'class_classes') args += ' --dev %s ' % GpuListCuda self.runJob(CUDA_ALIGN_SIGNIFICANT, args, numberOfMpi=1) copy( join(fnDir, "level_%02d" % (self.class2dIterations.get() - 1), "images.xmd"), join(fnDir, "images.xmd")) # After classification the stk and xmd files should be produced classesXmd = join( fnDir, "level_%02d/class_classes.xmd" % (self.class2dIterations.get() - 1)) classesStk = join( fnDir, "level_%02d/class_classes.stk" % (self.class2dIterations.get() - 1)) else: fnDir = self._getExtraPath("direction_%s" % projNumber) if not exists(join(fnDir, "level_00")): makePath(fnDir) # Run CL2D classification for the images assigned to one direction args = "-i %s " % fnToUse args += "--odir %s " % fnDir args += "--ref0 %s --iter %d --nref %d " % \ (projRef, self.class2dIterations, Nclasses) args += "--distance correlation --classicalMultiref " args += "--maxShift %f " % self.maxShift try: self.runJob("xmipp_classify_CL2D", args, numberOfMpi=self.numberOfMpi.get() * self.numberOfThreads.get()) except: return # After CL2D the stk and xmd files should be produced classesXmd = join(fnDir, "level_%02d/class_classes.xmd" % Nlevels) classesStk = join(fnDir, "level_%02d/class_classes.stk" % Nlevels) # Let's check that the output was produced if not exists(classesStk): return # Run align of the class average and the projection representative fnAlignRoot = join(fnDir, "classes") args = "-i %s " % classesStk args += "--ref %s " % projRef args += " --oroot %s --iter 1" % fnAlignRoot self.runJob("xmipp_image_align", args, numberOfMpi=1) # Apply alignment args = "-i %s_alignment.xmd --apply_transform" % fnAlignRoot self.runJob("xmipp_transform_geometry", args, numberOfMpi=1) for classNo in range(1, Nclasses + 1): localImagesMd = emlib.MetaData("class%06d_images@%s" % (classNo, classesXmd)) # New class detected self.classCount += 1 # Check which images have not been assigned yet to any class # and assign them to this new class for objId in localImagesMd: imgId = localImagesMd.getValue(emlib.MDL_ITEM_ID, objId) # Add images not classify yet and store their class number if imgId not in self.classImages: self.classImages.add(imgId) newObjId = mdImages.addObject() mdImages.setValue(emlib.MDL_ITEM_ID, imgId, newObjId) mdImages.setValue(emlib.MDL_REF2, self.classCount, newObjId) newClassId = mdClasses.addObject() mdClasses.setValue(emlib.MDL_REF, projNumber, newClassId) mdClasses.setValue(emlib.MDL_REF2, self.classCount, newClassId) mdClasses.setValue(emlib.MDL_IMAGE, "%d@%s" % (classNo, classesStk), newClassId) mdClasses.setValue(emlib.MDL_IMAGE1, projRef, newClassId) mdClasses.setValue(emlib.MDL_CLASS_COUNT, localImagesMd.size(), newClassId)
def test_forward_backwards(self): """convert transformation matrix to xmipp and back""" mList = [ [ [0.71461016, 0.63371837, -0.29619813, 1.], # a1 [-0.61309201, 0.77128059, 0.17101008, 2.], [0.33682409, 0.059391174, 0.93969262, 3.], [0, 0, 0, 1.] ], [ [0., 0., -1., 0.], # a2 [0., 1., 0., 0.], [1., 0., 0., 0.], [0., 0., 0., 1.] ], [ [0., 1., 0., 0.], # a3 [0., 0., 1., 0.], [1., 0., 0., 0.], [0., 0., 0., 1.] ], [ [0.22612257, 0.82379508, -0.51983678, 0.], # a4 [-0.88564873, 0.39606407, 0.24240388, 0.], [0.40557978, 0.40557978, 0.81915206, 0.], [0., 0., 0., 1.] ], [ [-0.78850311, -0.24329656, -0.56486255, 0.], # a5 [0.22753462, -0.96866286, 0.099600501, 0.], [-0.57139379, -0.049990479, 0.81915206, 0.], [0., 0., 0., 1.] ], [ [1.0, 0.0, 0.0, 0.0], # a6 [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0] ], [ [0., 0., -1., 0.], # a7 [-1., 0., 0., 0.], [0., 1., 0., 0.], [0., 0., 0., 1.] ] ] aList = [np.array(m) for m in mList] rowa = md.Row() rowb = md.Row() labelList = [ md.RLN_ORIENT_ROT, md.RLN_ORIENT_TILT, md.RLN_ORIENT_PSI, md.RLN_ORIENT_ORIGIN_X, md.RLN_ORIENT_ORIGIN_Y, md.RLN_ORIENT_ORIGIN_Z ] for i, a in enumerate(aList): a = Transform(aList[i]) convert.alignmentToRow(a, rowa, ALIGN_PROJ) b = convert.rowToAlignment(rowa, ALIGN_PROJ) convert.alignmentToRow(b, rowb, ALIGN_PROJ) # same two matrices self.assertTrue( np.allclose(a.getMatrix(), b.getMatrix(), rtol=1e-2)) for label in labelList: auxBtilt = rowb.getValue(label) auxAtilt = rowa.getValue(label) # same two rows self.assertAlmostEqual(auxBtilt, auxAtilt, places=3) b = convert.rowToAlignment(rowa, ALIGN_PROJ) convert.alignmentToRow(b, rowb, ALIGN_PROJ) aMatrix = a.getMatrix() # aMatrix[0,:] *= -1; aMatrix[2,:] *= -1; # same two matrices with flip print("\naMatrix: \n", aMatrix, "\nbMatrix: \n", b.getMatrix()) self.assertTrue(np.allclose(aMatrix, b.getMatrix(), rtol=1e-2))
def runMapBack(self, classId): for tomo in self.inputTomograms.get().iterItems(): if self.selection == 0: TsSubtomo = self.inputClasses.get().getSamplingRate() else: TsSubtomo = self.inputRef.get().getSamplingRate() TsTomo = tomo.getSamplingRate() scaleFactor = TsSubtomo / TsTomo mdGeometry = lib.MetaData() if self.selection == 0: inputSet = self.inputClasses.get().getFirstItem() ref = self._getExtraPath("reference%d.mrc" % classId) else: inputSet = self.inputSubtomos.get() ref = self.inputRef.get().getFileName() for subtomo in inputSet.iterItems(): fn = subtomo.getFileName() if fn.endswith('.mrc'): fn += ':mrc' if subtomo.getCoordinate3D().getVolId() == tomo.getObjId() \ or basename(subtomo.getVolName()) == tomo.getBaseName().partition('import_')[0]: nRow = md.Row() nRow.setValue(lib.MDL_ITEM_ID, int(subtomo.getObjId())) coord = subtomo.getCoordinate3D() coord.setVolume(tomo) nRow.setValue( lib.MDL_XCOOR, int( coord.getX(const.BOTTOM_LEFT_CORNER) * scaleFactor)) nRow.setValue( lib.MDL_YCOOR, int( coord.getY(const.BOTTOM_LEFT_CORNER) * scaleFactor)) nRow.setValue( lib.MDL_ZCOOR, int( coord.getZ(const.BOTTOM_LEFT_CORNER) * scaleFactor)) # Compute inverse matrix A = subtomo.getTransform().getMatrix() subtomo.getTransform().setMatrix(np.linalg.inv(A)) # Convert transform matrix to Euler Angles (rot, tilt, psi) alignmentToRow(subtomo.getTransform(), nRow, ALIGN_3D) nRow.addToMd(mdGeometry) fnGeometry = self._getExtraPath("geometry%d.xmd" % classId) mdGeometry.write(fnGeometry) if TsSubtomo != TsTomo: factor = TsSubtomo / TsTomo args = "-i %s -o %s --scale %d" % (ref, ref, factor) self.runJob('xmipp_transform_geometry', args) if self.paintingType.get() == 0: painting = 'copy' elif self.paintingType.get() == 1: painting = 'avg %d' % self.threshold.get() elif self.paintingType.get() == 2: painting = 'highlight %d' % self.constant.get() elif self.paintingType.get() == 3: painting = 'copy_binary %f' % self.threshold.get() tomogram = self._getExtraPath("tomogram_%d.mrc" % tomo.getObjId()) args = " -i %s -o %s --geom %s --ref %s --method %s" % ( tomogram, tomogram, self._getExtraPath("geometry%d.xmd" % classId), ref, painting) self.runJob("xmipp_tomo_map_back", args)