def importCoordinates(self, fileName, addCoordinate): if exists(fileName): ext = getExt(fileName) if ext == ".json": print "importando un json" jsonPosDict = loadJson(fileName) if jsonPosDict.has_key("boxes"): boxes = jsonPosDict["boxes"] for box in boxes: x, y = box[:2] coord = Coordinate() coord.setPosition(x, y) addCoordinate(coord) elif ext == ".box": md = MetaData() md.readPlain(fileName, 'xcoor ycoor xSize ySize') for objId in md: x = md.getValue(MDL_XCOOR, objId) y = md.getValue(MDL_YCOOR, objId) coord = Coordinate() coord.setPosition(x, y) addCoordinate(coord) else: print "importando un queseyo"
def importCoordinates(self, fileName, addCoordinate): if exists(fileName): ext = getExt(fileName) if ext == ".json": jsonPosDict = loadJson(fileName) if jsonPosDict.has_key("boxes"): boxes = jsonPosDict["boxes"] for box in boxes: x, y = box[:2] coord = Coordinate() coord.setPosition(x, y) addCoordinate(coord) elif ext == ".box": md = MetaData() md.readPlain(fileName, "xcoor ycoor particleSize") size = md.getValue(MDL_PICKING_PARTICLE_SIZE, md.firstObject()) if size is None: print ">>> WARNING: Error parsing coordinate file: %s" % fileName print " Skipping this file." else: half = size / 2 for objId in md: x = md.getValue(MDL_XCOOR, objId) y = md.getValue(MDL_YCOOR, objId) coord = Coordinate() coord.setPosition(x+half, y+half) addCoordinate(coord) else: raise Exception('Unknown extension "%s" to import Eman coordinates' % ext)
def importCoordinates(self, fileName, addCoordinate): print "In importCoordinates Appion with filename=%s" % fileName if exists(fileName): md = MetaData() md.readPlain(fileName, 'xcoor ycoor') for objId in md: x = md.getValue(MDL_XCOOR, objId) y = md.getValue(MDL_YCOOR, objId) coord = Coordinate() coord.setPosition(x, y) addCoordinate(coord)
def createOutputStep(self): inputParticles = self.inputParticles.get() inputMics = self.inputMicrographs.get() outputCoords = self._createSetOfCoordinates(inputMics) scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() #FIXME: the 'correction' as a temporarly hack for fixing the # coordinates scale according to the particles pixel size if self.correction > 0: scale = scale * self.correction.get() print "Scaling coordinates by a factor *%0.2f*" % scale newCoord = Coordinate() firstCoord = inputParticles.getFirstItem().getCoordinate() hasMicName = firstCoord.getMicName() is not None # Create the micrographs dict using either micName or micId micDict = {} for mic in inputMics: micKey = mic.getMicName() if hasMicName else mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!!!" % micKey) micDict[micKey] = mic.clone() for particle in inputParticles: coord = particle.getCoordinate() micKey = coord.getMicName() if hasMicName else coord.getMicId() mic = micDict.get(micKey, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) #FIXME: the 'correction' as a temporarly hack for fixing the # coordinates scale according to the particles pixel size x, y = coord.getPosition() newCoord.setPosition(x * scale, y * scale) newCoord.setMicrograph(mic) outputCoords.append(newCoord) boxSize = inputParticles.getXDim() * scale outputCoords.setBoxSize(boxSize) self._defineOutputs(outputCoordinates=outputCoords) self._defineSourceRelation(self.inputParticles, outputCoords) self._defineSourceRelation(self.inputMicrographs, outputCoords)
def createOutputStep(self): inputParticles = self.inputParticles.get() inputMics = self.inputMicrographs.get() outputCoords = self._createSetOfCoordinates(inputMics) scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() # FIXME: the 'correction' as a temporarly hack for fixing the # coordinates scale according to the particles pixel size if self.correction > 0: scale_corrected = scale * self.correction.get() print "Scaling coordinates by a factor *%0.2f*" % scale newCoord = Coordinate() firstCoord = inputParticles.getFirstItem().getCoordinate() hasMicName = firstCoord.getMicName() is not None # Create the micrographs dict using either micName or micId micDict = {} for mic in inputMics: micKey = mic.getMicName() if hasMicName else mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!!!" % micKey) micDict[micKey] = mic.clone() for particle in inputParticles: coord = particle.getCoordinate() micKey = coord.getMicName() if hasMicName else coord.getMicId() mic = micDict.get(micKey, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) # FIXME: the 'correction' as a temporarly hack for fixing the # coordinates scale according to the particles pixel size x, y = coord.getPosition() newCoord.setPosition(x * scale_corrected, y * scale_corrected) newCoord.setMicrograph(mic) outputCoords.append(newCoord) boxSize = inputParticles.getXDim() * scale outputCoords.setBoxSize(boxSize) self._defineOutputs(outputCoordinates=outputCoords) self._defineSourceRelation(self.inputParticles, outputCoords) self._defineSourceRelation(self.inputMicrographs, outputCoords)
def readCoordinates(mic, fileName, coordsSet): if exists(fileName): jsonPosDict = loadJson(fileName) if jsonPosDict.has_key("boxes"): boxes = jsonPosDict["boxes"] for box in boxes: x, y = box[:2] coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordsSet.append(coord)
def createOutputStep(self): inputParticles = self.inputParticles.get() inputMics = self.inputMicrographs.get() outputCoords = self._createSetOfCoordinates(inputMics) alignType = inputParticles.getAlignment() scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() print "Scaling coordinates by a factor *%0.2f*" % scale newCoord = Coordinate() firstCoord = inputParticles.getFirstItem().getCoordinate() hasMicName = firstCoord.getMicName() is not None # Create the micrographs dict using either micName or micId micDict = {} for mic in inputMics: micKey = mic.getMicName() if hasMicName else mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!!!" % micKey) micDict[micKey] = mic.clone() for particle in inputParticles: coord = particle.getCoordinate() micKey = coord.getMicName() if hasMicName else coord.getMicId() mic = micDict.get(micKey, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) x, y = coord.getPosition() if self.applyShifts: shifts = self.getShifts(particle.getTransform(), alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) newCoord.setPosition(xCoor * scale, yCoor * scale) else: newCoord.setPosition(x * scale, y * scale) newCoord.setMicrograph(mic) outputCoords.append(newCoord) boxSize = inputParticles.getXDim() * scale outputCoords.setBoxSize(boxSize) self._defineOutputs(outputCoordinates=outputCoords) self._defineSourceRelation(self.inputParticles, outputCoords) self._defineSourceRelation(self.inputMicrographs, outputCoords)
def createOutputStep(self): inputParticles = self.inputParticles.get() inputMics = self.inputMicrographs.get() outputCoords = self._createSetOfCoordinates(inputMics) alignType = inputParticles.getAlignment() scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() print "Scaling coordinates by a factor *%0.2f*" % scale newCoord = Coordinate() firstCoord = inputParticles.getFirstItem().getCoordinate() hasMicName = firstCoord.getMicName() is not None # Create the micrographs dict using either micName or micId micDict = {} for mic in inputMics: micKey = mic.getMicName() if hasMicName else mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!!!" % micKey) micDict[micKey] = mic.clone() for particle in inputParticles: coord = particle.getCoordinate() micKey = coord.getMicName() if hasMicName else coord.getMicId() mic = micDict.get(micKey, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) x, y = coord.getPosition() if self.applyShifts: shifts = self.getShifts(particle.getTransform(), alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) newCoord.setPosition(xCoor*scale, yCoor*scale) else: newCoord.setPosition(x*scale, y*scale) newCoord.setMicrograph(mic) outputCoords.append(newCoord) boxSize = inputParticles.getXDim() * scale outputCoords.setBoxSize(boxSize) self._defineOutputs(outputCoordinates=outputCoords) self._defineSourceRelation(self.inputParticles, outputCoords) self._defineSourceRelation(self.inputMicrographs, outputCoords)
def readCoordinates(mic, fileName, coordsSet, invertY=False): if pwutils.exists(fileName): jsonPosDict = loadJson(fileName) if jsonPosDict.has_key("boxes"): boxes = jsonPosDict["boxes"] for box in boxes: x, y = box[:2] if invertY: y = mic.getYDim() - y coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordsSet.append(coord)
def _fillParticles(self, outputParticles, outputCoords, inputParticles): """ Create the SetOfParticles and SetOfCoordinates""" myParticles = md.MetaData(self._getExtraPath('final_images.xmd')) outputParticles.enableAppend() outputCoords.enableAppend() #Calculating the scale that relates the coordinates with the actual # position in the mic scale = inputParticles.getSamplingRate() / \ self.inputMics.get().getSamplingRate() #Dictionary with the name and id of the inpt mics micDictname = {} micDictId = {} for mic in self.inputMics.get(): micKey = mic.getMicName() micDictname[micKey] = mic.clone() micKey2 = mic.getObjId() micDictId[micKey2] = mic.clone() for row in md.iterRows(myParticles): #To create the new particle p = rowToParticle(row) outputParticles.append(p) #To create the new coordinate newCoord = Coordinate() coord = p.getCoordinate() if coord.getMicName() is not None: micKey = coord.getMicName() micDict = micDictname else: micKey = coord.getMicId() micDict = micDictId mic = micDict.get(micKey, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(p) x, y = coord.getPosition() newCoord.setPosition(x * scale, y * scale) newCoord.setMicrograph(mic) outputCoords.append(newCoord) boxSize = inputParticles.getXDim() * scale outputCoords.setBoxSize(boxSize)
def readCoordsFromMics(self, workingDir, micList, coordSet): coordSet.setBoxSize(self.radius.get() * 2) for mic in micList: micFn = mic.getFileName() micDir = self._getMicDir(micFn) coordFile = os.path.join(micDir, self._getMicPosFn(micFn)) if os.path.exists(coordFile): coordMd = md.MetaData() coordMd.readPlain(coordFile, 'xcoor ycoor') for objId in coordMd: x = coordMd.getValue(md.MDL_XCOOR, objId) y = coordMd.getValue(md.MDL_YCOOR, objId) coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordSet.append(coord) else: print "Coordinate file '%s' not found. " % coordFile
def readSetOfCoordinates(self, workingDir, coordSet): for mic in self.getInputMicrographs(): micFn = mic.getFileName() micDir = self._getMicDir(micFn) coordFile = os.path.join(micDir, self._getMicPosFn(micFn)) if os.path.exists(coordFile): coordMd = md.MetaData() coordMd.readPlain(coordFile, 'xcoor ycoor') for objId in coordMd: x = coordMd.getValue(md.MDL_XCOOR, objId) y = coordMd.getValue(md.MDL_YCOOR, objId) coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordSet.append(coord) else: print "Coordinate file '%s' not found. " % coordFile
def calculateConsensusStep(self, micrograph): for coordinates in self.inputCoordinates: while len([ x.getPosition() for x in coordinates.get().iterCoordinates( micrograph.getObjId()) ]) < 1: time.sleep(1) # Take the sampling rates Tm = [] for coordinates in self.inputCoordinates: Tm.append(coordinates.get().getMicrographs().getSamplingRate()) # Get all coordinates for this micrograph coords = [] Ncoords = 0 n = 0 for coordinates in self.inputCoordinates: coordArray = np.asarray([ x.getPosition() for x in coordinates.get().iterCoordinates( micrograph.getObjId()) ], dtype=float) coordArray *= float(Tm[n]) / float(Tm[0]) coords.append(np.asarray(coordArray, dtype=int)) Ncoords += coordArray.shape[0] n += 1 allCoords = np.zeros([Ncoords, 2]) votes = np.zeros(Ncoords) # Add all coordinates in the first method N0 = coords[0].shape[0] inAllMicrographs = (self.consensus <= 0 or self.consensus >= len(self.inputCoordinates)) if N0 == 0 and inAllMicrographs: return elif N0 > 0: allCoords[0:N0, :] = coords[0] votes[0:N0] = 1 # Add the rest of coordinates Ncurrent = N0 for n in range(1, len(self.inputCoordinates)): for coord in coords[n]: if Ncurrent > 0: dist = np.sum((coord - allCoords[0:Ncurrent])**2, axis=1) imin = np.argmin(dist) if sqrt(dist[imin]) < self.consensusRadius: newCoord = (votes[imin] * allCoords[imin, ] + coord) / (votes[imin] + 1) allCoords[imin, ] = newCoord votes[imin] += 1 else: allCoords[Ncurrent, :] = coord votes[Ncurrent] = 1 Ncurrent += 1 else: allCoords[Ncurrent, :] = coord votes[Ncurrent] = 1 Ncurrent += 1 # Select those in the consensus if self.consensus <= 0 or self.consensus > len(self.inputCoordinates): consensus = len(self.inputCoordinates) else: consensus = self.consensus.get() consensusCoords = allCoords[votes >= consensus, :] try: jaccardIdx = float(len(consensusCoords)) / ( float(len(allCoords)) / len(self.inputCoordinates)) # COSS: Possible problem with concurrent writes with open(self._getExtraPath('jaccard.txt'), "a") as fhJaccard: fhJaccard.write("%d %f\n" % (micrograph.getObjId(), jaccardIdx)) except: pass # Write the consensus file only if there # are some coordinates (size > 0) if consensusCoords.size: np.savetxt( self._getExtraPath('consensus_%06d.txt' % micrograph.getObjId()), consensusCoords) fnTmp = self._getExtraPath('consensus_%06d.txt' % micrograph.getObjId()) if os.path.exists(fnTmp): coords = np.loadtxt(fnTmp) if coords.size == 2: # special case with only one coordinate in consensus coords = [coords] for coord in coords: aux = Coordinate() aux.setMicrograph(micrograph) aux.setPosition(coord[0], coord[1]) self.setOfCoords.append(aux) self.mics.append(micrograph)
def convertInputStep(self): inputParticles = self.inputParticles.get() firstCoord = inputParticles.getFirstItem().getCoordinate() self.hasMicName = firstCoord.getMicName() is not None inputMics = self._getMicrographs() self.alignType = inputParticles.getAlignment() self.downFactor = self.ctfDownFactor.get() # create a tmp set for matching mics self.matchingMics = self._createSetOfMicrographs(suffix='_tmp') self.matchingMics.copyInfo(inputMics) if self.downFactor != 1.: self.matchingMics.setDownsample(self.downFactor) # create a tmp set for coords coords = self._createSetOfCoordinates(inputMics, suffix='_tmp') newCoord = Coordinate() self.scale = inputParticles.getSamplingRate( ) / inputMics.getSamplingRate() if self.scale != 1.0: print "Scaling coordinates by a factor *%0.2f*" % self.scale # Create the micrograph dicts micDict = {} # dict with micName or micId micBaseDict = {} # dict with micName (just basename) micKey2 = None insertedMics = {} for mic in inputMics: if self.hasMicName: micKey = mic.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!" % micKey) micDict[micKey] = mic.clone() if self.hasMicName: micBaseDict[micKey2] = mic.clone() # match the mic from coord with micDict for particle in inputParticles: coord = particle.getCoordinate() or None if coord is None: print "Skipping particle, coordinates not found" continue if self.hasMicName: micKey = coord.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = coord.getMicId() # find the mapping by micName (with or without ext) or micId mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) x, y = coord.getPosition() if self.applyShifts: shifts = getShifts(particle.getTransform(), self.alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) newCoord.setPosition(xCoor * self.scale, yCoor * self.scale) else: newCoord.setPosition(x * self.scale, y * self.scale) newCoord.setMicrograph(mic) coords.append(newCoord) if mic.getObjId() not in insertedMics: insertedMics[mic.getObjId()] = mic self.matchingMics.append(mic) ih = em.ImageHandler() # We convert matching micrographs if they are not *.mrc for mic in self.matchingMics: # Create micrograph dir micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if self.downFactor != 1.: ih.scaleFourier(micName, outMic, self.downFactor) sps = inputMics.getScannedPixelSize() * self.downFactor self._params['scannedPixelSize'] = sps else: if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(micName, outMic) # Write out coordinate files and sets writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics) coords.clear() pwutils.cleanPath(coords.getFileName()) self.matchingMics.write() self.matchingMics.close()
def convertInputStep(self): inputParticles = self.inputParticles.get() firstCoord = inputParticles.getFirstItem().getCoordinate() self.hasMicName = firstCoord.getMicName() is not None inputMics = self._getMicrographs() self.alignType = inputParticles.getAlignment() self.downFactor = self.ctfDownFactor.get() # create a tmp set for matching mics self.matchingMics = self._createSetOfMicrographs(suffix='_tmp') self.matchingMics.copyInfo(inputMics) if self.downFactor != 1.: self.matchingMics.setDownsample(self.downFactor) # create a tmp set for coords coords = self._createSetOfCoordinates(inputMics, suffix='_tmp') newCoord = Coordinate() self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() if self.scale != 1.0: print "Scaling coordinates by a factor *%0.2f*" % self.scale # Create the micrograph dicts micDict = {} # dict with micName or micId micBaseDict = {} # dict with micName (just basename) micKey2 = None insertedMics = {} for mic in inputMics: if self.hasMicName: micKey = mic.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!" % micKey) micDict[micKey] = mic.clone() if self.hasMicName: micBaseDict[micKey2] = mic.clone() # match the mic from coord with micDict for particle in inputParticles: coord = particle.getCoordinate() or None if coord is None: print "Skipping particle, coordinates not found" continue if self.hasMicName: micKey = coord.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = coord.getMicId() # find the mapping by micName (with or without ext) or micId mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) x, y = coord.getPosition() if self.applyShifts: shifts = getShifts(particle.getTransform(), self.alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) newCoord.setPosition(xCoor * self.scale, yCoor * self.scale) else: newCoord.setPosition(x * self.scale, y * self.scale) newCoord.setMicrograph(mic) coords.append(newCoord) if mic.getObjId() not in insertedMics: insertedMics[mic.getObjId()] = mic self.matchingMics.append(mic) ih = em.ImageHandler() # We convert matching micrographs if they are not *.mrc for mic in self.matchingMics: # Create micrograph dir micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if self.downFactor != 1.: ih.scaleFourier(micName, outMic, self.downFactor) sps = inputMics.getScannedPixelSize() * self.downFactor self._params['scannedPixelSize'] = sps else: if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(micName, outMic) # Write out coordinate files and sets writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics) coords.clear() pwutils.cleanPath(coords.getFileName()) self.matchingMics.write() self.matchingMics.close()