def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(
self._getPath(inputFit),
self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(
self._getPath(
'movie_particles_shiny_%sclass001_unfil.mrc' %
half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
half))
self._renameFiles('movie_particles_shiny_post*',
'movie_particles_')
self._renameFiles('movie_particles_shiny*',
'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(
mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameter just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
imgStarTmp = self._getTmpPath('movie_particles.star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStarTmp))
writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(),
alignType=imgSet.getAlignment(),
extraLabels=MOVIE_EXTRA_LABELS)
mdImg = md.MetaData(imgStarTmp)
# replace mdColumn from *.stk to *.mrcs as Relion2 requires
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
from convert import relionToLocation, locationToRelion
for objId in mdImg:
index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId))
if not imgPath.endswith('mrcs'):
newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs')
newPath = self._getTmpPath(newName)
newLoc = locationToRelion(index, newPath)
if not exists(newPath):
pwutils.createLink(imgPath, newPath)
mdImg.setValue(mdColumn, newLoc, objId)
mdImg.write(imgStar)
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = em.Particle()
for mic in micList:
# We need to make this dict because there is no ID in the
# coord.star file
coordDict = {}
for coord in self.coordDict[mic.getObjId()]:
coordDict[self._getPos(coord)] = coord
_, partStarFn = self._getStarFiles(mic)
for row in md.iterRows(self._getPath(partStarFn)):
pos = (row.getValue(md.RLN_IMAGE_COORD_X),
row.getValue(md.RLN_IMAGE_COORD_Y))
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME))
p.setLocation(idx, self._getPath(fn[2:]))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
outputParts.append(p)
# Release the list of coordinates for this micrograph since it
# will not be longer needed
del self.coordDict[mic.getObjId()]
def _loadClassesInfo(self):
""" Read some information about the produced Relion 3D classes
from the *model.star file.
"""
self._classesInfo = {} # store classes info, indexed by class id
mdModel = self._getFileName('outputModel', lev=self._level)
modelStar = md.MetaData('model_classes@' + mdModel)
for classNumber, row in enumerate(md.iterRows(modelStar)):
index, fn = relionToLocation(row.getValue('rlnReferenceImage'))
# Store info indexed by id, we need to store the row.clone() since
# the same reference is used for iteration
self._classesInfo[classNumber + 1] = (index, fn, row.clone())
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameter just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
imgStarTmp = self._getTmpPath('movie_particles.star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStarTmp))
writeSetOfParticles(imgSet,
imgStarTmp,
self._getExtraPath(),
alignType=imgSet.getAlignment(),
extraLabels=MOVIE_EXTRA_LABELS)
mdImg = md.MetaData(imgStarTmp)
# replace mdColumn from *.stk to *.mrcs as Relion2 requires
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
from convert import relionToLocation, locationToRelion
for objId in mdImg:
index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId))
if not imgPath.endswith('mrcs'):
newName = pwutils.replaceBaseExt(os.path.basename(imgPath),
'mrcs')
newPath = self._getTmpPath(newName)
newLoc = locationToRelion(index, newPath)
if not exists(newPath):
pwutils.createLink(imgPath, newPath)
mdImg.setValue(mdColumn, newLoc, objId)
mdImg.write(imgStar)
def organizeDataStep(self):
from pyworkflow.utils import moveFile
import pyworkflow.em.metadata as md
from convert import relionToLocation, locationToRelion
# moving shiny.star form project base path to the current protocol extra path.
shinyStar = "shiny.star"
pathFixedShiny = self._getExtraPath(shinyStar)
if exists(shinyStar):
moveFile(shinyStar, pathFixedShiny)
mdShiny = md.MetaData(pathFixedShiny)
oldImgPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = self._getExtraPath(os.path.basename(imgPath))
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldImgPath != imgPath and exists(imgPath):
moveFile(imgPath, newPath)
oldImgPath = imgPath
mdShiny.write(pathFixedShiny)
def organizeDataStep(self):
from pyworkflow.utils import moveFile
import pyworkflow.em.metadata as md
from convert import relionToLocation, locationToRelion
# moving shiny.star form project base path to the current protocol extra path.
shinyStar = "shiny.star"
pathFixedShiny = self._getExtraPath(shinyStar)
if exists(shinyStar):
moveFile(shinyStar, pathFixedShiny)
mdShiny = md.MetaData(pathFixedShiny)
oldImgPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = self._getExtraPath(os.path.basename(imgPath))
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldImgPath != imgPath and exists(imgPath):
moveFile(imgPath, newPath)
oldImgPath = imgPath
mdShiny.write(pathFixedShiny)
def _updateItem(self, item, row):
newFn = row.getValue(md.RLN_IMAGE_NAME)
newLoc = relionToLocation(newFn)
item.setLocation(newLoc)
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = em.Particle()
# Let's create a dict with the names of the mic in Relion star files
# and also create a set with all star files to iterate them once
starSet = set()
micPathDict = {}
for mic in micList:
starSet.add(self._micCoordStarDict[mic.getObjId()])
micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc')
micFile = os.path.join('extra', micName)
micPathDict[micFile] = mic
prevMicFile = None
mic = None
for partStarFn in starSet:
for row in md.iterRows(self._getPath(partStarFn),
sortByLabel=md.RLN_MICROGRAPH_NAME):
micFile = row.getValue(md.RLN_MICROGRAPH_NAME)
if micFile != prevMicFile: # Load some stuff when new mic
if prevMicFile is not None: # cleanup previous mic
# Release the list of coordinates for this micrograph
# since it will not be longer needed
del self.coordDict[mic.getObjId()]
mic = micPathDict[micFile]
coordDict = {self._getPos(coord): coord
for coord in self.coordDict[mic.getObjId()]}
posSet = set() # Set of (x, y) pairs to avoid duplicates
prevMicFile = micFile
pos = (row.getValue(md.RLN_IMAGE_COORD_X),
row.getValue(md.RLN_IMAGE_COORD_Y))
if pos in posSet:
print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
coord = None
else:
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME))
p.setLocation(idx, self._getPath(fn[2:]))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
outputParts.append(p)
posSet.add(pos)
# Clean up the last mic if necessary
if mic is not None:
del self.coordDict[mic.getObjId()]
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = em.Particle()
# Let's create a dict with the names of the mic in Relion star files
# and also create a set with all star files to iterate them once
starSet = set()
micPathDict = {}
for mic in micList:
starSet.add(self._micCoordStarDict[mic.getObjId()])
micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc')
micFile = os.path.join('extra', micName)
micPathDict[micFile] = mic
prevMicFile = None
mic = None
for partStarFn in starSet:
for row in md.iterRows(self._getPath(partStarFn),
sortByLabel=md.RLN_MICROGRAPH_NAME):
micFile = row.getValue(md.RLN_MICROGRAPH_NAME)
if micFile != prevMicFile: # Load some stuff when new mic
if prevMicFile is not None: # cleanup previous mic
# Release the list of coordinates for this micrograph
# since it will not be longer needed
del self.coordDict[mic.getObjId()]
mic = micPathDict[micFile]
coordDict = {self._getPos(coord): coord
for coord in self.coordDict[mic.getObjId()]}
posSet = set() # Set of (x, y) pairs to avoid duplicates
prevMicFile = micFile
pos = (row.getValue(md.RLN_IMAGE_COORD_X),
row.getValue(md.RLN_IMAGE_COORD_Y))
if pos in posSet:
print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
coord = None
else:
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME))
p.setLocation(idx, self._getPath(fn[2:]))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
outputParts.append(p)
posSet.add(pos)
# Clean up the last mic if necessary
if mic is not None:
del self.coordDict[mic.getObjId()]
def _updateItem(self, item, row):
newFn = row.getValue(md.RLN_IMAGE_NAME)
newLoc = relionToLocation(newFn)
item.setLocation(newLoc)
def _updateItem(self, particle, row):
newLoc = convert.relionToLocation(row.getValue('rlnImageName'))
particle.setLocation(newLoc)
def _plotClassDistribution(self, paramName=None):
labels = ["rlnClassDistribution", "rlnAccuracyRotations",
"rlnAccuracyTranslations"]
iterations = range(self.firstIter, self.lastIter + 1)
classInfo = {}
for it in iterations:
modelStar = self.protocol._getFileName('model', iter=it)
for row in md.iterRows('%s@%s' % ('model_classes', modelStar)):
i, fn = relionToLocation(row.getValue('rlnReferenceImage'))
if i == em.NO_INDEX: # the case for 3D classes
# NOTE: Since there is not an proper ID value in
# the clases metadata, we are assuming that class X
# has a filename *_classXXX.mrc (as it is in Relion)
# and we take the ID from there
index = int(fn[-7:-4])
else:
index = i
if index not in classInfo:
classInfo[index] = {}
for l in labels:
classInfo[index][l] = []
for l in labels:
classInfo[index][l].append(row.getValue(l))
xplotter = VolSelPlotter()
xplotter.createSubPlot("Classes distribution over iterations",
"Iterations", "Classes Distribution")
# Empty list for each iteration
iters = [[]] * len(iterations)
l = labels[0]
for index in sorted(classInfo.keys()):
for it, value in enumerate(classInfo[index][l]):
iters[it].append(value)
ax = xplotter.getLastSubPlot()
n = len(iterations)
ind = range(n)
bottomValues = [0] * n
width = 0.45 # the width of the bars: can also be len(x) sequence
def get_cmap(N):
import matplotlib.cm as cmx
import matplotlib.colors as colors
"""Returns a function that maps each index in 0, 1, ... N-1
to a distinct RGB color."""
color_norm = colors.Normalize(vmin=0, vmax=N)#-1)
scalar_map = cmx.ScalarMappable(norm=color_norm, cmap='hsv')
def map_index_to_rgb_color(index):
return scalar_map.to_rgba(index)
return map_index_to_rgb_color
cmap = get_cmap(len(classInfo))
for classId in sorted(classInfo.keys()):
values = classInfo[classId][l]
ax.bar(ind, values, width, label='class %s' % classId,
bottom=bottomValues, color=cmap(classId))
bottomValues = [a+b for a, b in zip(bottomValues, values)]
ax.get_xaxis().set_ticks([i + 0.25 for i in ind])
ax.get_xaxis().set_ticklabels([str(i) for i in ind])
ax.legend(loc='upper left', fontsize='xx-small')
return [xplotter]
