def createOutputStep(self):
setEnviron() # Set the environment to access localrec modules
import localrec
import pyrelion
# return
inputSet = self._getInputParticles()
outputSet = self._createSetOfCoordinates(inputSet)
params = {
"symmetryGroup": self.symmetryGroup.get(),
"vector": self.vector.get(),
"vectorFile": self.vectorFile.get(),
"length": self.length.get(),
"unique": self.unique.get(),
"mindist": self.mindist.get(),
"side": self.side.get(),
"top": self.top.get(),
"pxSize": self.inputParticles.get().getSamplingRate(),
"dim": self.inputParticles.get().getXDim()
}
symMatrices = localrec.matrix_from_symmetry(self.symmetryGroup.get())
if self.defineVector == CMM:
cmmFn = params["vectorFile"]
vector = " "
else:
cmmFn = ""
vector = params["vector"]
subpartVectorList = localrec.load_vectors(cmmFn, vector,
params["length"],
params["pxSize"])
# Define some conditions to filter subparticles
filters = localrec.load_filters(math.radians(params["side"]),
math.radians(params["top"]),
params["mindist"])
coord = Coordinate()
for part in inputSet:
partItem = pyrelion.Item()
particleToRow(part, partItem)
subparticles = self.getSubparticles(localrec, partItem,
symMatrices, params,
subpartVectorList, filters)
for subpart in subparticles:
rowToSubcoordinate(subpart, coord, part)
coord.setObjId(None) # Force to insert as a new item
outputSet.append(coord)
if part.hasAttribute('_rlnRandomSubset'):
coord._subparticle.copyAttributes(part, '_rlnRandomSubset')
restituteRelionHome()
self._defineOutputs(outputCoordinates=outputSet)
self._defineSourceRelation(self.inputParticles, outputSet)
def createOutputStep(self):
setEnviron() # Set the environment to access localrec modules
import localrec
import pyrelion
# return
inputSet = self._getInputParticles()
outputSet = self._createSetOfCoordinates(inputSet)
params = {"symmetryGroup" : self.symmetryGroup.get(),
"vector" : self.vector.get(),
"vectorFile" : self.vectorFile.get(),
"length" : self.length.get(),
"unique" : self.unique.get(),
"mindist" : self.mindist.get(),
"side" : self.side.get(),
"top" : self.top.get(),
"pxSize" : self.inputParticles.get().getSamplingRate(),
"dim" : self.inputParticles.get().getXDim()
}
symMatrices = localrec.matrix_from_symmetry(self.symmetryGroup.get())
if self.defineVector == CMM:
cmmFn = params["vectorFile"]
vector = " "
else:
cmmFn = ""
vector = params["vector"]
subpartVectorList = localrec.load_vectors(cmmFn, vector,
params["length"],
params["pxSize"])
# Define some conditions to filter subparticles
filters = localrec.load_filters(math.radians(params["side"]),
math.radians(params["top"]),
params["mindist"])
coord = Coordinate()
for part in inputSet:
partItem = pyrelion.Item()
particleToRow(part, partItem)
subparticles = self.getSubparticles(localrec, partItem,
symMatrices, params,
subpartVectorList, filters)
for subpart in subparticles:
rowToSubcoordinate(subpart, coord, part)
coord.setObjId(None) # Force to insert as a new item
outputSet.append(coord)
if part.hasAttribute('_rlnRandomSubset'):
coord._subparticle.copyAttributes(part, '_rlnRandomSubset')
restituteRelionHome()
self._defineOutputs(outputCoordinates=outputSet)
self._defineSourceRelation(self.inputParticles, outputSet)
def importCoordinates(self, fileName, addCoordinate):
print "In importCoordinates Appion with filename=%s" % fileName
if exists(fileName):
md = MetaData()
md.readPlain(fileName, 'xcoor ycoor')
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
addCoordinate(coord)
def importCoordinates(self, fileName, addCoordinate):
print "In importCoordinates Appion with filename=%s" % fileName
if exists(fileName):
md = MetaData()
md.readPlain(fileName, 'xcoor ycoor')
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
addCoordinate(coord)
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
alignType = inputParticles.getAlignment()
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = self.getShifts(particle.getTransform(), alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * scale, yCoor * scale)
else:
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def createOutputStep(self):
firstCoords = self.inputCoordinates[0].get()
inputMics = firstCoords.getMicrographs()
setOfCoordinates = self._createSetOfCoordinates(inputMics)
setOfCoordinates.setBoxSize(firstCoords.getBoxSize())
# Read all consensus particles
for micrograph in inputMics:
fnTmp = self._getExtraPath('consensus_%06d.txt' %
micrograph.getObjId())
if os.path.exists(fnTmp):
coords = np.loadtxt(fnTmp)
if coords.size == 2: # special case with only one coordinate in consensus
coords = [coords]
for coord in coords:
aux = Coordinate()
aux.setMicrograph(micrograph)
aux.setX(coord[0])
aux.setY(coord[1])
setOfCoordinates.append(aux)
#cleanPath(fnTmp)
# Set output
self._defineOutputs(outputCoordinates=setOfCoordinates)
for coordinates in self.inputCoordinates:
self._defineSourceRelation(coordinates, self.outputCoordinates)
def importCoordinates(self, fileName, addCoordinate):
if exists(fileName):
ext = getExt(fileName)
if ext == ".json":
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
coord = Coordinate()
coord.setPosition(x, y)
addCoordinate(coord)
elif ext == ".box":
md = MetaData()
md.readPlain(fileName, "xcoor ycoor particleSize")
size = md.getValue(MDL_PICKING_PARTICLE_SIZE, md.firstObject())
if size is None:
print ">>> WARNING: Error parsing coordinate file: %s" % fileName
print " Skipping this file."
else:
half = size / 2
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x+half, y+half)
addCoordinate(coord)
else:
raise Exception('Unknown extension "%s" to import Eman coordinates' % ext)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
alignType = inputParticles.getAlignment()
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = self.getShifts(particle.getTransform(), alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor*scale, yCoor*scale)
else:
newCoord.setPosition(x*scale, y*scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def createOutputStep(self):
firstCoords = self.inputCoordinates[0].get()
inputMics = firstCoords.getMicrographs()
setOfCoordinates = self._createSetOfCoordinates(inputMics)
setOfCoordinates.setBoxSize(firstCoords.getBoxSize())
# Read all consensus particles
for micrograph in inputMics:
fnTmp = self._getExtraPath("consensus_%06d.txt" % micrograph.getObjId())
if os.path.exists(fnTmp):
coords = np.loadtxt(fnTmp)
if coords.size == 2: # special case with only one coordinate in consensus
coords = [coords]
for coord in coords:
aux = Coordinate()
aux.setMicrograph(micrograph)
aux.setX(coord[0])
aux.setY(coord[1])
setOfCoordinates.append(aux)
# cleanPath(fnTmp)
# Set output
self._defineOutputs(outputCoordinates=setOfCoordinates)
for coordinates in self.inputCoordinates:
self._defineSourceRelation(coordinates.get(), self.outputCoordinates)
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
#FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
if self.correction > 0:
scale = scale * self.correction.get()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
#FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
x, y = coord.getPosition()
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def importCoordinates(self, fileName, addCoordinate):
if exists(fileName):
ext = getExt(fileName)
if ext == ".json":
print "importando un json"
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
coord = Coordinate()
coord.setPosition(x, y)
addCoordinate(coord)
elif ext == ".box":
md = MetaData()
md.readPlain(fileName, 'xcoor ycoor xSize ySize')
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
addCoordinate(coord)
else:
print "importando un queseyo"
def createOutputStep(self):
inputParticles = self.inputParticles.get()
inputMics = self.inputMicrographs.get()
outputCoords = self._createSetOfCoordinates(inputMics)
scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
# FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
if self.correction > 0:
scale_corrected = scale * self.correction.get()
print "Scaling coordinates by a factor *%0.2f*" % scale
newCoord = Coordinate()
firstCoord = inputParticles.getFirstItem().getCoordinate()
hasMicName = firstCoord.getMicName() is not None
# Create the micrographs dict using either micName or micId
micDict = {}
for mic in inputMics:
micKey = mic.getMicName() if hasMicName else mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!!!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!!!" % micKey)
micDict[micKey] = mic.clone()
for particle in inputParticles:
coord = particle.getCoordinate()
micKey = coord.getMicName() if hasMicName else coord.getMicId()
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
# FIXME: the 'correction' as a temporarly hack for fixing the
# coordinates scale according to the particles pixel size
x, y = coord.getPosition()
newCoord.setPosition(x * scale_corrected, y * scale_corrected)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
self._defineOutputs(outputCoordinates=outputCoords)
self._defineSourceRelation(self.inputParticles, outputCoords)
self._defineSourceRelation(self.inputMicrographs, outputCoords)
def _fillParticles(self, outputParticles, outputCoords, inputParticles):
""" Create the SetOfParticles and SetOfCoordinates"""
myParticles = md.MetaData(self._getExtraPath('final_images.xmd'))
outputParticles.enableAppend()
outputCoords.enableAppend()
#Calculating the scale that relates the coordinates with the actual
# position in the mic
scale = inputParticles.getSamplingRate() / \
self.inputMics.get().getSamplingRate()
#Dictionary with the name and id of the inpt mics
micDictname = {}
micDictId = {}
for mic in self.inputMics.get():
micKey = mic.getMicName()
micDictname[micKey] = mic.clone()
micKey2 = mic.getObjId()
micDictId[micKey2] = mic.clone()
for row in md.iterRows(myParticles):
#To create the new particle
p = rowToParticle(row)
outputParticles.append(p)
#To create the new coordinate
newCoord = Coordinate()
coord = p.getCoordinate()
if coord.getMicName() is not None:
micKey = coord.getMicName()
micDict = micDictname
else:
micKey = coord.getMicId()
micDict = micDictId
mic = micDict.get(micKey, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(p)
x, y = coord.getPosition()
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
boxSize = inputParticles.getXDim() * scale
outputCoords.setBoxSize(boxSize)
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def extractCoordinates(self, partsIds=None):
inPart = self.getInputParticles()
inMics = self.getInputMicrographs()
scale = inPart.getSamplingRate() / inMics.getSamplingRate()
print "Scaling coordinates by a factor *%0.2f*" % scale
alignType = inPart.getAlignment()
suffix = self.getSuffix(partsIds[0]) if partsIds is not None else ''
outputCoords = self._createSetOfCoordinates(inMics, suffix=suffix)
def appendCoordFromParticle(part):
coord = part.getCoordinate()
micKey = coord.getMicId()
mic = inMics[micKey]
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(part)
x, y = coord.getPosition()
if self.applyShifts:
shifts = self.getShifts(part.getTransform(), alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * scale, yCoor * scale)
else:
newCoord.setPosition(x * scale, y * scale)
newCoord.setMicrograph(mic)
outputCoords.append(newCoord)
newCoord = Coordinate()
if self.streamingModeOn:
for partId in partsIds:
particle = inPart[partId]
appendCoordFromParticle(particle)
else:
for particle in inPart:
appendCoordFromParticle(particle)
boxSize = inPart.getXDim() * scale
outputCoords.setBoxSize(boxSize)
return outputCoords
def aaatest_particlesToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles.star')
fnStk = self.getOutputPath('particles.stk')
print ">>> Writing to file: %s" % fnStar
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def readCoordinates(mic, fileName, coordsSet, invertY=False):
if pwutils.exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
if invertY:
y = mic.getYDim() - y
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def test_particlesWithPhaseShiftToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(filename=self.getOutputPath("particles_ph_sh.sqlite"))
n = 10
fn = self.getFile('particles_binary')
ctfs = [CTFModel(defocusU=10000, defocusV=15000,
defocusAngle=15, phaseShift=90),
CTFModel(defocusU=20000, defocusV=25000,
defocusAngle=25, phaseShift=60)
]
acquisition = Acquisition(magnification=60000, voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles_ph_sh.star')
fnStk = self.getOutputPath('particles.stk')
print (">>> Writing to file: %s" % fnStar)
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
self.assertTrue(mdAll.containsLabel(md.RLN_CTF_PHASESHIFT))
def readCoordsFromMics(self, workingDir, micList, coordSet):
coordSet.setBoxSize(self.radius.get() * 2)
for mic in micList:
micFn = mic.getFileName()
micDir = self._getMicDir(micFn)
coordFile = os.path.join(micDir, self._getMicPosFn(micFn))
if os.path.exists(coordFile):
coordMd = md.MetaData()
coordMd.readPlain(coordFile, 'xcoor ycoor')
for objId in coordMd:
x = coordMd.getValue(md.MDL_XCOOR, objId)
y = coordMd.getValue(md.MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordSet.append(coord)
else:
print "Coordinate file '%s' not found. " % coordFile
def readSetOfCoordinates(self, workingDir, coordSet):
for mic in self.getInputMicrographs():
micFn = mic.getFileName()
micDir = self._getMicDir(micFn)
coordFile = os.path.join(micDir, self._getMicPosFn(micFn))
if os.path.exists(coordFile):
coordMd = md.MetaData()
coordMd.readPlain(coordFile, 'xcoor ycoor')
for objId in coordMd:
x = coordMd.getValue(md.MDL_XCOOR, objId)
y = coordMd.getValue(md.MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordSet.append(coord)
else:
print "Coordinate file '%s' not found. " % coordFile
def calculateConsensusStep(self, micrograph):
for coordinates in self.inputCoordinates:
while len([
x.getPosition() for x in coordinates.get().iterCoordinates(
micrograph.getObjId())
]) < 1:
time.sleep(1)
# Take the sampling rates
Tm = []
for coordinates in self.inputCoordinates:
Tm.append(coordinates.get().getMicrographs().getSamplingRate())
# Get all coordinates for this micrograph
coords = []
Ncoords = 0
n = 0
for coordinates in self.inputCoordinates:
coordArray = np.asarray([
x.getPosition() for x in coordinates.get().iterCoordinates(
micrograph.getObjId())
],
dtype=float)
coordArray *= float(Tm[n]) / float(Tm[0])
coords.append(np.asarray(coordArray, dtype=int))
Ncoords += coordArray.shape[0]
n += 1
allCoords = np.zeros([Ncoords, 2])
votes = np.zeros(Ncoords)
# Add all coordinates in the first method
N0 = coords[0].shape[0]
inAllMicrographs = (self.consensus <= 0
or self.consensus >= len(self.inputCoordinates))
if N0 == 0 and inAllMicrographs:
return
elif N0 > 0:
allCoords[0:N0, :] = coords[0]
votes[0:N0] = 1
# Add the rest of coordinates
Ncurrent = N0
for n in range(1, len(self.inputCoordinates)):
for coord in coords[n]:
if Ncurrent > 0:
dist = np.sum((coord - allCoords[0:Ncurrent])**2, axis=1)
imin = np.argmin(dist)
if sqrt(dist[imin]) < self.consensusRadius:
newCoord = (votes[imin] * allCoords[imin, ] +
coord) / (votes[imin] + 1)
allCoords[imin, ] = newCoord
votes[imin] += 1
else:
allCoords[Ncurrent, :] = coord
votes[Ncurrent] = 1
Ncurrent += 1
else:
allCoords[Ncurrent, :] = coord
votes[Ncurrent] = 1
Ncurrent += 1
# Select those in the consensus
if self.consensus <= 0 or self.consensus > len(self.inputCoordinates):
consensus = len(self.inputCoordinates)
else:
consensus = self.consensus.get()
consensusCoords = allCoords[votes >= consensus, :]
try:
jaccardIdx = float(len(consensusCoords)) / (
float(len(allCoords)) / len(self.inputCoordinates))
# COSS: Possible problem with concurrent writes
with open(self._getExtraPath('jaccard.txt'), "a") as fhJaccard:
fhJaccard.write("%d %f\n" %
(micrograph.getObjId(), jaccardIdx))
except:
pass
# Write the consensus file only if there
# are some coordinates (size > 0)
if consensusCoords.size:
np.savetxt(
self._getExtraPath('consensus_%06d.txt' %
micrograph.getObjId()), consensusCoords)
fnTmp = self._getExtraPath('consensus_%06d.txt' %
micrograph.getObjId())
if os.path.exists(fnTmp):
coords = np.loadtxt(fnTmp)
if coords.size == 2: # special case with only one coordinate in consensus
coords = [coords]
for coord in coords:
aux = Coordinate()
aux.setMicrograph(micrograph)
aux.setPosition(coord[0], coord[1])
self.setOfCoords.append(aux)
self.mics.append(micrograph)
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate(
) / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale,
yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir,
pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
def create_subparticles(particle, symmetry_matrices, subparticle_vector_list,
part_image_size, randomize, subparticles_total,
align_subparticles):
""" Obtain all subparticles from a given particle and set
the properties of each such subparticle. """
# Euler angles that take particle to the orientation of the model
matrix_particle = inv(particle.getTransform().getMatrix())
shifts, angles = geometryFromMatrix(matrix_particle)
subparticles = []
subparticles_total += 1
symmetry_matrix_ids = range(1, len(symmetry_matrices) + 1)
if randomize:
# randomize the order of symmetry matrices, prevents preferred views
random.shuffle(symmetry_matrix_ids)
for subparticle_vector in subparticle_vector_list:
matrix_from_subparticle_vector = subparticle_vector.get_matrix()
for symmetry_matrix_id in symmetry_matrix_ids:
# symmetry_matrix_id can be later written out to find out
# which symmetry matrix created this subparticle
symmetry_matrix = np.array(symmetry_matrices[symmetry_matrix_id -
1][0:3, 0:3])
subpart = particle.clone()
m = np.matmul(
matrix_particle[0:3, 0:3],
(np.matmul(symmetry_matrix.transpose(),
matrix_from_subparticle_vector.transpose())))
angles_org = -1.0 * np.ones(3) * euler_from_matrix(m, 'szyz')
if align_subparticles:
angles = -1.0 * np.ones(3) * euler_from_matrix(m, 'szyz')
else:
m2 = np.matmul(matrix_particle[0:3, 0:3],
symmetry_matrix.transpose())
angles = -1.0 * np.ones(3) * euler_from_matrix(m2, 'szyz')
# subparticle origin
d = subparticle_vector.get_length()
x = -m[0, 2] * d + shifts[0]
y = -m[1, 2] * d + shifts[1]
z = -m[2, 2] * d
# save the subparticle coordinates (integer part) relative to the
# user given image size and as a small shift in the origin (decimal part)
x_d, x_i = math.modf(x)
y_d, y_i = math.modf(y)
alignment = em.Transform()
alignmentOrg = em.Transform()
M = matrixFromGeometry(np.array([-x_d, -y_d, 0]), angles, True)
MOrg = matrixFromGeometry(np.array([-x_d, -y_d, 0]), angles_org,
True)
alignment.setMatrix(M)
alignmentOrg.setMatrix(MOrg)
subpart._transorg = alignmentOrg.clone()
subpart.setTransform(alignment)
coord = Coordinate()
coord.setObjId(None)
coord.setX(int(part_image_size / 2) - x_i)
coord.setY(int(part_image_size / 2) - y_i)
coord.setMicId(particle.getObjId())
if subpart.hasCTF():
ctf = subpart.getCTF()
ctf.setDefocusU(subpart.getDefocusU() + z)
ctf.setDefocusV(subpart.getDefocusV() + z)
subpart.setCoordinate(coord)
coord._subparticle = subpart.clone()
subparticles.append(subpart)
return subparticles
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale, yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
def _particlesFromEmx(protocol,
emxData,
emxFile,
outputDir,
acquisition,
samplingRate,
copyOrLink,
alignType=ALIGN_NONE):
""" Create the output SetOfCoordinates or SetOfParticles given an EMXData object.
Add CTF information to the particles if present.
"""
emxParticle = emxData.getFirstObject(emxlib.PARTICLE)
partDir = dirname(emxFile)
micSet = getattr(protocol, 'outputMicrographs', None)
if emxParticle is not None:
# Check if there are particles or coordinates
fn = emxParticle.get(emxlib.FILENAME)
if exists(join(
partDir,
fn)): # if the particles has binary data, means particles case
partSet = protocol._createSetOfParticles()
partSet.setAcquisition(
acquisition) # this adquisition is per project
#we need one per particle
part = Particle()
if _hasCtfLabels(emxParticle) or _hasCtfLabels(
emxParticle.getMicrograph()):
part.setCTF(CTFModel())
partSet.setHasCTF(True)
if emxParticle.has('centerCoord__X'):
part.setCoordinate(Coordinate())
#if emxParticle.has('transformationMatrix__t11'):
# _particleFromEmx(emxParticle, part)
# #partSet.setAlignment3D()
# partSet.setAlignment(alignType)
#else:
# partSet.setAlignment(alignType)
partSet.setAlignment(alignType)
if not samplingRate:
samplingRate = part.getSamplingRate()
partSet.setSamplingRate(samplingRate)
partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
particles = True
else: # if not binary data, the coordinate case
if micSet is None:
raise Exception(
'Could not import Coordinates from EMX, micrographs not imported.'
)
partSet = protocol._createSetOfCoordinates(micSet)
part = Coordinate()
particles = False
copiedFiles = {} # copied or linked
for emxParticle in emxData.iterClasses(emxlib.PARTICLE):
if particles:
_particleFromEmx(emxParticle, part)
i, fn = part.getLocation()
partFn = join(partDir, fn)
newFn = join(outputDir, basename(partFn))
newLoc = (i, newFn)
if not partFn in copiedFiles:
copyOrLink(partFn, newFn)
copiedFiles[partFn] = newFn
part.setLocation(newLoc)
if partSet.hasAlignment():
transform = Transform()
_transformFromEmx(emxParticle, part, transform, alignType)
part.setTransform(transform)
else:
_coordinateFromEmx(emxParticle, part)
partSet.append(part)
part.cleanObjId()
if particles:
protocol._defineOutputs(outputParticles=partSet)
else:
protocol._defineOutputs(outputCoordinates=partSet)
if micSet is not None:
protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
