def _particlesToEmx(emxData, partSet, micSet=None, **kwargs): """ Write a SetOfMicrograph as expected in EMX format Params: micSet: input set of micrographs filename: the EMX file where to store the micrographs information. micSet: micrographs set associated with the particles **kwargs: writeImages: if set to False, only coordinates are exported. imagesStack: if passed all images will be output into a single stack file. imagesPrefix: used when not imagesStack is passed. A different stack will be created per micrograph. """ writeImages = kwargs.get('writeImages', True) imagesPrefix = kwargs.get('imagesPrefix', None) micDict = {} # Use singleMic for count all particles to be written to a single stack imagesStack = kwargs.get('imagesStack', None) singleMic = Micrograph() singleMic.setFileName(imagesStack) singleMic.counter = pwobj.Integer(0) def _getMicKey(particle): coord = particle.getCoordinate() if coord is None or coord.getMicName() is None: return '%05d' % particle.getMicId() else: return pwutils.removeExt(coord.getMicName()) def _getLocation(particle): if imagesStack is not None: mic = singleMic else: micKey = _getMicKey(particle) if micKey not in micDict: mic = Micrograph() mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey)) mic.counter = pwobj.Integer(0) micDict[micKey] = mic else: mic = micDict[micKey] # Count one more particle assigned to this micrograph mic.counter.increment() return (mic.counter.get(), mic.getFileName()) ih = ImageHandler() partAlign = partSet.getAlignment() for particle in partSet: if writeImages: newLoc = _getLocation(particle) ih.convert(particle, newLoc) localFn = basename(newLoc[1]) particle.setLocation(newLoc[0], localFn) emxObj = _particleToEmx(emxData, particle, micSet, partAlign) else: emxObj = _coordinateToEmx(emxData, particle, micSet) emxData.addObject(emxObj)
def _particlesToEmx(emxData, partSet, micSet=None, **kwargs): """ Write a SetOfMicrograph as expected in EMX format Params: micSet: input set of micrographs filename: the EMX file where to store the micrographs information. micSet: micrographs set associated with the particles **kwargs: writeImages: if set to False, only coordinates are exported. imagesStack: if passed all images will be output into a single stack file. imagesPrefix: used when not imagesStack is passed. A different stack will be created per micrograph. """ writeImages = kwargs.get('writeImages', True) imagesPrefix = kwargs.get('imagesPrefix', None) micDict = {} # Use singleMic for count all particles to be written to a single stack imagesStack = kwargs.get('imagesStack', None) singleMic = Micrograph() singleMic.setFileName(imagesStack) singleMic.counter = pwobj.Integer(0) def _getMicKey(particle): coord = particle.getCoordinate() if coord is None or coord.getMicName() is None: return '%05d' % particle.getMicId() else: return pwutils.removeExt(coord.getMicName()) def _getLocation(particle): if imagesStack is not None: mic = singleMic else: micKey = _getMicKey(particle) if micKey not in micDict: mic = Micrograph() mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey)) mic.counter = pwobj.Integer(0) micDict[micKey] = mic else: mic = micDict[micKey] # Count one more particle assigned to this micrograph mic.counter.increment() return (mic.counter.get(), mic.getFileName()) ih = ImageHandler() partAlign = partSet.getAlignment() for particle in partSet: if writeImages: newLoc = _getLocation(particle) ih.convert(particle, newLoc) localFn = basename(newLoc[1]) particle.setLocation(newLoc[0], localFn) emxObj = _particleToEmx(emxData, particle, micSet, partAlign) else: emxObj = _coordinateToEmx(emxData, particle, micSet) emxData.addObject(emxObj)
def _preprocessImageRow(self, img, imgRow): from convert import setupCTF, copyOrLinkFileName if self._imgPath is not None: copyOrLinkFileName(imgRow, self._imgPath, self.protocol._getExtraPath()) setupCTF(imgRow, self.protocol.samplingRate.get()) if self._micIdOrName: micId = imgRow.getValue('rlnMicrographId', None) micName = imgRow.getValue('rlnMicrographName', None) # Check which is the key to identify micrographs (id or name) if micId is not None: micKey = micId else: micKey = micName mic = self.micDict.get(micKey, None) # First time I found this micrograph (either by id or name) if mic is None: mic = Micrograph() mic.setObjId(micId) if micName is None: micName = self.protocol._getExtraPath( 'fake_micrograph%6d' % micId) mic.setFileName(micName) mic.setMicName(os.path.basename(micName)) self.micSet.append(mic) # Update dict with new Micrograph self.micDict[micKey] = mic # Update the row to set a MDL_MICROGRAPH_ID imgRow.setValue('rlnMicrographId', long(mic.getObjId()))
def _checkNewMics(self, micSet): """ Check for already computed CTF and update the output set. """ micDict = {} newMic = False for mic in micSet: micDict[mic.getFileName()] = True if micDict: if micSet.getSize(): micSet.enableAppend() micSet.loadAllProperties() else: micSet.setStreamState(micSet.STREAM_OPEN) acquisition = Acquisition() acquisition.setMagnification(self._magnification) acquisition.setVoltage(self._voltage) acquisition.setSphericalAberration(self._sphericalAberration) acquisition.setAmplitudeContrast(self._amplitudeContrast) micSet.setAcquisition(acquisition) micSet.setSamplingRate(self.samplingRate.get()) mic = Micrograph() counter = 0 for k, v in self.xmippMic.iteritems(): counter += 1 if (k not in micDict): mic.setFileName(k) mic.setMicName(basename(k)) mic.setObjId(counter) micSet.append(mic) newMic = True return micSet, newMic #why a dictionary, a boolean may be enought
def _preprocessImageRow(self, img, imgRow): from convert import setupCTF, copyOrLinkFileName if self._imgPath is not None: copyOrLinkFileName(imgRow, self._imgPath, self.protocol._getExtraPath()) setupCTF(imgRow, self.protocol.samplingRate.get()) if self._micIdOrName: micId = imgRow.getValue('rlnMicrographId', None) micName = imgRow.getValue('rlnMicrographName', None) # Check which is the key to identify micrographs (id or name) if micId is not None: micKey = micId else: micKey = micName mic = self.micDict.get(micKey, None) # First time I found this micrograph (either by id or name) if mic is None: mic = Micrograph() mic.setObjId(micId) if micName is None: micName = self.protocol._getExtraPath('fake_micrograph%6d' % micId) mic.setFileName(micName) mic.setMicName(os.path.basename(micName)) self.micSet.append(mic) # Update dict with new Micrograph self.micDict[micKey] = mic # Update the row to set a MDL_MICROGRAPH_ID imgRow.setValue('rlnMicrographId', long(mic.getObjId()))
def _getLocation(particle): if imagesStack is not None: mic = singleMic else: micKey = _getMicKey(particle) if micKey not in micDict: mic = Micrograph() mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey)) mic.counter = pwobj.Integer(0) micDict[micKey] = mic else: mic = micDict[micKey] # Count one more particle assigned to this micrograph mic.counter.increment() return (mic.counter.get(), mic.getFileName())
def _checkNewItems(self, objSet): """ Check for already computed micrograph/movie and update the output set. """ objDict = {} newObj = False for obj in objSet: objDict[obj.getFileName()] = True if objDict: if objSet.getSize(): objSet.enableAppend() objSet.loadAllProperties() else: objSet.setStreamState(objSet.STREAM_OPEN) acquisition = Acquisition() acquisition.setMagnification(self._magnification) acquisition.setVoltage(self._voltage) acquisition.setSphericalAberration(self._sphericalAberration) acquisition.setAmplitudeContrast(self._amplitudeContrast) objSet.setAcquisition(acquisition) objSet.setSamplingRate(self.samplingRate.get()) if self.setof == SET_OF_MOVIES: obj = Movie() elif self.setof == SET_OF_MICROGRAPHS: obj = Micrograph() elif self.setof == SET_OF_RANDOM_MICROGRAPHS: obj = Micrograph() else: raise Exception('Unknown data type') counter = 0 for k, v in self.dictObj.iteritems(): counter += 1 if (k not in objDict): obj.setFileName(k) obj.setMicName(basename(k)) obj.setObjId(counter) objSet.append(obj) newObj = True return objSet, newObj # why a dictionary, a boolean may be enought
def _preprocessParticleRow(self, img, imgRow): if self._imgPath: # Create a link or copy files to extraPath # and update the Row properly index, fn = xmippToLocation(imgRow.getValue(md.MDL_IMAGE)) imgBase = basename(fn) imgDst = self.protocol._getExtraPath(imgBase) if not exists(imgDst): self.copyOrLink(join(self._imgPath, fn), imgDst) imgRow.setValue(md.MDL_IMAGE, locationToXmipp(index, imgDst)) if self._micIdOrName: micId = imgRow.getValue(md.MDL_MICROGRAPH_ID, None) micName = imgRow.getValue(md.MDL_MICROGRAPH, None) # Check which is the key to identify micrographs (id or name) if micId is not None: micKey = micId else: micKey = micName mic = self.micDict.get(micKey, None) # First time I found this micrograph (either by id or name) if mic is None: mic = Micrograph() mic.setObjId(micId) if micName is None: micName = self.protocol._getExtraPath( 'fake_micrograph%6d' % micId) mic.setFileName(micName) self.micSet.append(mic) # Update dict with new Micrograph self.micDict[micKey] = mic # Update the row to set a MDL_MICROGRAPH_ID imgRow.setValue(md.MDL_MICROGRAPH_ID, long(mic.getObjId())) # JMRT: This means that the metadata contains MDL_CTF_MODEL # and the files path were found from some root # In Xmipp 3.1 the ctfparam metadata in particles # was replaced with directly seeting the CTF values # so we need to fill those in the particle row if self._ctfPath: ctfModel = imgRow.getValue(md.MDL_CTF_MODEL) if ctfModel in self._ctfDict: ctfRow = self._ctfDict[ctfModel] else: ctfRow = md.Row() ctfRow.readFromFile(join(self._ctfPath, ctfModel)) self._ctfDict[ctfModel] = ctfRow imgRow.copyFromRow(ctfRow)
def _preprocessParticleRow(self, img, imgRow): if self._imgPath: # Create a link or copy files to extraPath # and update the Row properly index, fn = xmippToLocation(imgRow.getValue(md.MDL_IMAGE)) imgBase = basename(fn) imgDst = self.protocol._getExtraPath(imgBase) if not exists(imgDst): self.copyOrLink(join(self._imgPath, fn), imgDst) imgRow.setValue(md.MDL_IMAGE, locationToXmipp(index, imgDst)) if self._micIdOrName: micId = imgRow.getValue(md.MDL_MICROGRAPH_ID, None) micName = imgRow.getValue(md.MDL_MICROGRAPH, None) # Check which is the key to identify micrographs (id or name) if micId is not None: micKey = micId else: micKey = micName mic = self.micDict.get(micKey, None) # First time I found this micrograph (either by id or name) if mic is None: mic = Micrograph() mic.setObjId(micId) if micName is None: micName = self.protocol._getExtraPath('fake_micrograph%6d' % micId) mic.setFileName(micName) self.micSet.append(mic) # Update dict with new Micrograph self.micDict[micKey] = mic # Update the row to set a MDL_MICROGRAPH_ID imgRow.setValue(md.MDL_MICROGRAPH_ID, long(mic.getObjId())) # JMRT: This means that the metadata contains MDL_CTF_MODEL # and the files path were found from some root # In Xmipp 3.1 the ctfparam metadata in particles # was replaced with directly seeting the CTF values # so we need to fill those in the particle row if self._ctfPath: ctfModel = imgRow.getValue(md.MDL_CTF_MODEL) if ctfModel in self._ctfDict: ctfRow = self._ctfDict[ctfModel] else: ctfRow = md.Row() ctfRow.readFromFile(join(self._ctfPath, ctfModel)) self._ctfDict[ctfModel] = ctfRow imgRow.copyFromRow(ctfRow)
def _getLocation(particle): if imagesStack is not None: mic = singleMic else: micKey = _getMicKey(particle) if micKey not in micDict: mic = Micrograph() mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey)) mic.counter = pwobj.Integer(0) micDict[micKey] = mic else: mic = micDict[micKey] # Count one more particle assigned to this micrograph mic.counter.increment() return (mic.counter.get(), mic.getFileName())
def _preprocessParticleRow(self, img, imgRow): if self._imgPath: # Create a link or copy files to extraPath # and update the Row properly index, fn = xmippToLocation(imgRow.getValue(md.MDL_IMAGE)) imgBase = basename(fn) imgDst = self.protocol._getExtraPath(imgBase) if not exists(imgDst): self.copyOrLink(join(self._imgPath, fn), imgDst) imgRow.setValue(md.MDL_IMAGE, locationToXmipp(index, imgDst)) if self._micIdOrName: micId = imgRow.getValue(md.MDL_MICROGRAPH_ID, None) micName = imgRow.getValue(md.MDL_MICROGRAPH, None) # Check which is the key to identify micrographs (id or name) if micId is not None: micKey = micId else: micKey = micName mic = self.micDict.get(micKey, None) # First time I found this micrograph (either by id or name) if mic is None: mic = Micrograph() mic.setObjId(micId) if micName is None: micName = self.protocol._getExtraPath('fake_micrograph%6d' % micId) mic.setFileName(micName) self.micSet.append(mic) # Update dict with new Micrograph self.micDict[micKey] = mic # Update the row to set a MDL_MICROGRAPH_ID imgRow.setValue(md.MDL_MICROGRAPH_ID, long(mic.getObjId()))
def testCtfdiscrepancyWorkflow(self): # create one micrograph set fnMicSet = self.proj.getTmpPath("mics.sqlite") fnMic = self.proj.getTmpPath("mic.mrc") mic = Micrograph() mic.setFileName(fnMic) micSet = SetOfMicrographs(filename=fnMicSet) # create two CTFsets fnCTF1 = self.proj.getTmpPath("ctf1.sqlite") fnCTF2 = self.proj.getTmpPath("ctf2.sqlite") ctfSet1 = SetOfCTF(filename=fnCTF1) ctfSet2 = SetOfCTF(filename=fnCTF2) # create one fake micrographs image projSize = 32 img = xmipp.Image() img.setDataType(xmipp.DT_FLOAT) img.resize(projSize, projSize) img.write(fnMic) # fill the sets for i in range(1, 4): mic = Micrograph() mic.setFileName(fnMic) micSet.append(mic) defocusU = 1000+10*i defocusV = 1000+i defocusAngle = i*10 psdFile = "psd_1%04d" % i ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile) ctf.setMicrograph(mic) ctfSet1.append(ctf) defocusU = 1000+20*i defocusV = 1000+i defocusAngle = i*20 psdFile = "psd_2%04d" % i ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile) ctf.setMicrograph(mic) ctfSet2.append(ctf) ctfSet1.write() ctfSet2.write() micSet.write() # import micrograph set args = {'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION, 'sqliteFile': fnMicSet, 'amplitudConstrast': 0.1, 'sphericalAberration': 2., 'voltage': 100, 'samplingRate': 2.1 } protMicImport = self.newProtocol(ProtImportMicrographs, **args) protMicImport.setObjLabel('import micrographs from sqlite ') self.launchProtocol(protMicImport) # import ctfsets protCTF1 = \ self.newProtocol(ProtImportCTF, importFrom=ProtImportCTF.IMPORT_FROM_SCIPION, filesPath=fnCTF1) protCTF2 = \ self.newProtocol(ProtImportCTF, importFrom=ProtImportCTF.IMPORT_FROM_SCIPION, filesPath=fnCTF2) protCTF1.inputMicrographs.set(protMicImport.outputMicrographs) protCTF2.inputMicrographs.set(protMicImport.outputMicrographs) protCTF1.setObjLabel('import ctfs from scipion_1 ') protCTF2.setObjLabel('import ctfs from scipion_2 ') self.launchProtocol(protCTF1) self.launchProtocol(protCTF2) # launch CTF discrepancy protocol protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy) protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF) protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF) protCtfDiscrepancy.setObjLabel('ctf discrepancy') self.launchProtocol(protCtfDiscrepancy) ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem() resolution = int(ctf0.getResolution()) defocusU = int(ctf0.getDefocusU()) self.assertEqual(resolution, 2) self.assertEqual(defocusU, 1010)
def _micrographsFromEmx(protocol, emxData, emxFile, outputDir, acquisition, samplingRate, copyOrLink): """ Create the output SetOfMicrographs given an EMXData object. If there is information of the CTF, also the SetOfCTF will be registered as output of the protocol. """ emxMic = emxData.getFirstObject(emxlib.MICROGRAPH) if emxMic is not None: micSet = protocol._createSetOfMicrographs() mic = Micrograph() mic.setAcquisition(acquisition) if _hasCtfLabels(emxMic): mic.setCTF(CTFModel()) ctfSet = protocol._createSetOfCTF() else: ctfSet = None _micrographFromEmx(emxMic, mic) acq = mic.getAcquisition().clone() micSet.setAcquisition(acq) micSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get()) if not samplingRate: samplingRate = mic.getSamplingRate() micSet.setSamplingRate(samplingRate) micDir = dirname(emxFile) for emxMic in emxData.iterClasses(emxlib.MICROGRAPH): _micrographFromEmx(emxMic, mic) _, fn = mic.getLocation() micFn = join(micDir, fn) if copyOrLink is not None: micBase = basename(micFn) newFn = join(outputDir, micBase) copyOrLink(micFn, newFn) mic.setLocation(newFn) else: mic.setLocation(micFn) _fillMicName(mic, fn) micSet.append(mic) emxMic._micId = mic.getObjId() mic.cleanObjId() if ctfSet is not None: ctf = mic.getCTF().clone() ctf.setMicrograph(mic) #TODO I do not think next line is needed #ctf.setMicFile(newFn) ctfSet.append(ctf) if emxMic is not None: protocol._defineOutputs(outputMicrographs=micSet) if ctfSet is not None: protocol._defineOutputs(outputCTF=ctfSet) ctfSet.setMicrographs(micSet) protocol._defineCtfRelation(micSet, ctfSet)
def testCtfdiscrepancyWorkflow(self): # create one micrograph set fnMicSet = self.proj.getTmpPath("mics.sqlite") fnMic = self.proj.getTmpPath("mic.mrc") mic = Micrograph() mic.setFileName(fnMic) micSet = SetOfMicrographs(filename=fnMicSet) # create two CTFsets fnCTF1 = self.proj.getTmpPath("ctf1.sqlite") fnCTF2 = self.proj.getTmpPath("ctf2.sqlite") ctfSet1 = SetOfCTF(filename=fnCTF1) ctfSet2 = SetOfCTF(filename=fnCTF2) # create one fake micrographs image projSize = 32 img = xmipp.Image() img.setDataType(xmipp.DT_FLOAT) img.resize(projSize, projSize) img.write(fnMic) # fill the sets for i in range(1, 4): mic = Micrograph() mic.setFileName(fnMic) micSet.append(mic) defocusU = 1000 + 10 * i defocusV = 1000 + i defocusAngle = i * 10 psdFile = "psd_1%04d" % i ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile) ctf.setMicrograph(mic) ctfSet1.append(ctf) defocusU = 1000 + 20 * i defocusV = 1000 + i defocusAngle = i * 20 psdFile = "psd_2%04d" % i ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile) ctf.setMicrograph(mic) ctfSet2.append(ctf) ctfSet1.write() ctfSet2.write() micSet.write() # import micrograph set args = { 'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION, 'sqliteFile': fnMicSet, 'amplitudConstrast': 0.1, 'sphericalAberration': 2., 'voltage': 100, 'samplingRate': 2.1 } protMicImport = self.newProtocol(ProtImportMicrographs, **args) protMicImport.setObjLabel('import micrographs from sqlite ') self.launchProtocol(protMicImport) # import ctfsets protCTF1 = \ self.newProtocol(ProtImportCTF, importFrom=ProtImportCTF.IMPORT_FROM_SCIPION, filesPath=fnCTF1) protCTF2 = \ self.newProtocol(ProtImportCTF, importFrom=ProtImportCTF.IMPORT_FROM_SCIPION, filesPath=fnCTF2) protCTF1.inputMicrographs.set(protMicImport.outputMicrographs) protCTF2.inputMicrographs.set(protMicImport.outputMicrographs) protCTF1.setObjLabel('import ctfs from scipion_1 ') protCTF2.setObjLabel('import ctfs from scipion_2 ') self.launchProtocol(protCTF1) self.launchProtocol(protCTF2) # launch CTF discrepancy protocol protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy) protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF) protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF) protCtfDiscrepancy.setObjLabel('ctf discrepancy') self.launchProtocol(protCtfDiscrepancy) ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem() resolution = int(ctf0.getResolution()) defocusU = int(ctf0.getDefocusU()) self.assertEqual(resolution, 2) self.assertEqual(defocusU, 1010)
def _micrographsFromEmx(protocol, emxData, emxFile, outputDir, acquisition, samplingRate, copyOrLink): """ Create the output SetOfMicrographs given an EMXData object. If there is information of the CTF, also the SetOfCTF will be registered as output of the protocol. """ emxMic = emxData.getFirstObject(emxlib.MICROGRAPH) if emxMic is not None: micSet = protocol._createSetOfMicrographs() mic = Micrograph() mic.setAcquisition(acquisition) if _hasCtfLabels(emxMic): mic.setCTF(CTFModel()) ctfSet = protocol._createSetOfCTF() else: ctfSet = None _micrographFromEmx(emxMic, mic) acq = mic.getAcquisition().clone() micSet.setAcquisition(acq) micSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get()) if not samplingRate: samplingRate = mic.getSamplingRate() micSet.setSamplingRate(samplingRate) micDir = dirname(emxFile) for emxMic in emxData.iterClasses(emxlib.MICROGRAPH): _micrographFromEmx(emxMic, mic) _, fn = mic.getLocation() micFn = join(micDir, fn) if copyOrLink is not None: micBase = basename(micFn) newFn = join(outputDir, micBase) copyOrLink(micFn, newFn) mic.setLocation(newFn) else: mic.setLocation(micFn) _fillMicName(mic, fn) micSet.append(mic) emxMic._micId = mic.getObjId() mic.cleanObjId() if ctfSet is not None: ctf = mic.getCTF().clone() ctf.setMicrograph(mic) #TODO I do not think next line is needed #ctf.setMicFile(newFn) ctfSet.append(ctf) if emxMic is not None: protocol._defineOutputs(outputMicrographs=micSet) if ctfSet is not None: protocol._defineOutputs(outputCTF=ctfSet) ctfSet.setMicrographs(micSet) protocol._defineCtfRelation(micSet, ctfSet)