def createOutputStep(self):
firstCoords = self.inputCoordinates[0].get()
inputMics = firstCoords.getMicrographs()
setOfCoordinates = self._createSetOfCoordinates(inputMics)
setOfCoordinates.setBoxSize(firstCoords.getBoxSize())
# Read all consensus particles
for micrograph in inputMics:
fnTmp = self._getExtraPath('consensus_%06d.txt' %
micrograph.getObjId())
if os.path.exists(fnTmp):
coords = np.loadtxt(fnTmp)
if coords.size == 2: # special case with only one coordinate in consensus
coords = [coords]
for coord in coords:
aux = Coordinate()
aux.setMicrograph(micrograph)
aux.setX(coord[0])
aux.setY(coord[1])
setOfCoordinates.append(aux)
#cleanPath(fnTmp)
# Set output
self._defineOutputs(outputCoordinates=setOfCoordinates)
for coordinates in self.inputCoordinates:
self._defineSourceRelation(coordinates, self.outputCoordinates)
def createOutputStep(self):
firstCoords = self.inputCoordinates[0].get()
inputMics = firstCoords.getMicrographs()
setOfCoordinates = self._createSetOfCoordinates(inputMics)
setOfCoordinates.setBoxSize(firstCoords.getBoxSize())
# Read all consensus particles
for micrograph in inputMics:
fnTmp = self._getExtraPath("consensus_%06d.txt" % micrograph.getObjId())
if os.path.exists(fnTmp):
coords = np.loadtxt(fnTmp)
if coords.size == 2: # special case with only one coordinate in consensus
coords = [coords]
for coord in coords:
aux = Coordinate()
aux.setMicrograph(micrograph)
aux.setX(coord[0])
aux.setY(coord[1])
setOfCoordinates.append(aux)
# cleanPath(fnTmp)
# Set output
self._defineOutputs(outputCoordinates=setOfCoordinates)
for coordinates in self.inputCoordinates:
self._defineSourceRelation(coordinates.get(), self.outputCoordinates)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
# FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
if self.correction > 0:
scale_corrected = scale * self.correction.get()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
# FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
x, y = coord.getPosition()
newCoord.setPosition(x * scale_corrected, y * scale_corrected)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
#FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
if self.correction > 0:
scale = scale * self.correction.get()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
#FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
x, y = coord.getPosition()
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
alignType = inputParticles.getAlignment()
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = self.getShifts(particle.getTransform(), alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * scale, yCoor * scale)
else:
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
alignType = inputParticles.getAlignment()
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = self.getShifts(particle.getTransform(), alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor*scale, yCoor*scale)
else:
newCoord.setPosition(x*scale, y*scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def readCoordinates(mic, fileName, coordsSet, invertY=False):
if pwutils.exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
if invertY:
y = mic.getYDim() - y
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def _fillParticles(self, outputParticles, outputCoords, inputParticles):
""" Create the SetOfParticles and SetOfCoordinates"""
myParticles = md.MetaData(self._getExtraPath('final_images.xmd'))
outputParticles.enableAppend()
outputCoords.enableAppend()
#Calculating the scale that relates the coordinates with the actual
# position in the mic
scale = inputParticles.getSamplingRate() / \
self.inputMics.get().getSamplingRate()
#Dictionary with the name and id of the inpt mics
micDictname = {}
micDictId = {}
for mic in self.inputMics.get():
micKey = mic.getMicName()
micDictname[micKey] = mic.clone()
micKey2 = mic.getObjId()
micDictId[micKey2] = mic.clone()
for row in md.iterRows(myParticles):
#To create the new particle
p = rowToParticle(row)
outputParticles.append(p)
#To create the new coordinate
newCoord = Coordinate()
coord = p.getCoordinate()
if coord.getMicName() is not None:
micKey = coord.getMicName()
micDict = micDictname
else:
micKey = coord.getMicId()
micDict = micDictId
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(p)
x, y = coord.getPosition()
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
def readSetOfCoordinates(self, workingDir, coordSet):
for mic in self.getInputMicrographs():
micFn = mic.getFileName()
micDir = self._getMicDir(micFn)
coordFile = os.path.join(micDir, self._getMicPosFn(micFn))
if os.path.exists(coordFile):
coordMd = md.MetaData()
coordMd.readPlain(coordFile, 'xcoor ycoor')
for objId in coordMd:
x = coordMd.getValue(md.MDL_XCOOR, objId)
y = coordMd.getValue(md.MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordSet.append(coord)
else:
print "Coordinate file '%s' not found. " % coordFile
def readCoordsFromMics(self, workingDir, micList, coordSet):
coordSet.setBoxSize(self.radius.get() * 2)
for mic in micList:
micFn = mic.getFileName()
micDir = self._getMicDir(micFn)
coordFile = os.path.join(micDir, self._getMicPosFn(micFn))
if os.path.exists(coordFile):
coordMd = md.MetaData()
coordMd.readPlain(coordFile, 'xcoor ycoor')
for objId in coordMd:
x = coordMd.getValue(md.MDL_XCOOR, objId)
y = coordMd.getValue(md.MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordSet.append(coord)
else:
print "Coordinate file '%s' not found. " % coordFile
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale, yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate(
) / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale,
yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir,
pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
def calculateConsensusStep(self, micrograph):
for coordinates in self.inputCoordinates:
while len([
x.getPosition() for x in coordinates.get().iterCoordinates(
micrograph.getObjId())
]) < 1:
time.sleep(1)
# Take the sampling rates
Tm = []
for coordinates in self.inputCoordinates:
Tm.append(coordinates.get().getMicrographs().getSamplingRate())
# Get all coordinates for this micrograph
coords = []
Ncoords = 0
n = 0
for coordinates in self.inputCoordinates:
coordArray = np.asarray([
x.getPosition() for x in coordinates.get().iterCoordinates(
micrograph.getObjId())
],
dtype=float)
coordArray *= float(Tm[n]) / float(Tm[0])
coords.append(np.asarray(coordArray, dtype=int))
Ncoords += coordArray.shape[0]
n += 1
allCoords = np.zeros([Ncoords, 2])
votes = np.zeros(Ncoords)
# Add all coordinates in the first method
N0 = coords[0].shape[0]
inAllMicrographs = (self.consensus <= 0
or self.consensus >= len(self.inputCoordinates))
if N0 == 0 and inAllMicrographs:
return
elif N0 > 0:
allCoords[0:N0, :] = coords[0]
votes[0:N0] = 1
# Add the rest of coordinates
Ncurrent = N0
for n in range(1, len(self.inputCoordinates)):
for coord in coords[n]:
if Ncurrent > 0:
dist = np.sum((coord - allCoords[0:Ncurrent])**2, axis=1)
imin = np.argmin(dist)
if sqrt(dist[imin]) < self.consensusRadius:
newCoord = (votes[imin] * allCoords[imin, ] +
coord) / (votes[imin] + 1)
allCoords[imin, ] = newCoord
votes[imin] += 1
else:
allCoords[Ncurrent, :] = coord
votes[Ncurrent] = 1
Ncurrent += 1
else:
allCoords[Ncurrent, :] = coord
votes[Ncurrent] = 1
Ncurrent += 1
# Select those in the consensus
if self.consensus <= 0 or self.consensus > len(self.inputCoordinates):
consensus = len(self.inputCoordinates)
else:
consensus = self.consensus.get()
consensusCoords = allCoords[votes >= consensus, :]
try:
jaccardIdx = float(len(consensusCoords)) / (
float(len(allCoords)) / len(self.inputCoordinates))
# COSS: Possible problem with concurrent writes
with open(self._getExtraPath('jaccard.txt'), "a") as fhJaccard:
fhJaccard.write("%d %f\n" %
(micrograph.getObjId(), jaccardIdx))
except:
pass
# Write the consensus file only if there
# are some coordinates (size > 0)
if consensusCoords.size:
np.savetxt(
self._getExtraPath('consensus_%06d.txt' %
micrograph.getObjId()), consensusCoords)
fnTmp = self._getExtraPath('consensus_%06d.txt' %
micrograph.getObjId())
if os.path.exists(fnTmp):
coords = np.loadtxt(fnTmp)
if coords.size == 2: # special case with only one coordinate in consensus
coords = [coords]
for coord in coords:
aux = Coordinate()
aux.setMicrograph(micrograph)
aux.setPosition(coord[0], coord[1])
self.setOfCoords.append(aux)
self.mics.append(micrograph)
