def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile( self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile( self._getPath( 'movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation( mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation( mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def convertInputStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. Params: particlesId: use this parameter just to force redo of convert if the input particles are changed. """ imgSet = self._getInputParticles() imgStar = self._getFileName('movie_particles') imgStarTmp = self._getTmpPath('movie_particles.star') self.info("Converting set from '%s' into '%s'" % (imgSet.getFileName(), imgStarTmp)) writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(), alignType=imgSet.getAlignment(), extraLabels=MOVIE_EXTRA_LABELS) mdImg = md.MetaData(imgStarTmp) # replace mdColumn from *.stk to *.mrcs as Relion2 requires if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME from convert import relionToLocation, locationToRelion for objId in mdImg: index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId)) if not imgPath.endswith('mrcs'): newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs') newPath = self._getTmpPath(newName) newLoc = locationToRelion(index, newPath) if not exists(newPath): pwutils.createLink(imgPath, newPath) mdImg.setValue(mdColumn, newLoc, objId) mdImg.write(imgStar)
def readPartsFromMics(self, micList, outputParts): """ Read the particles extract for the given list of micrographs and update the outputParts set with new items. """ p = em.Particle() for mic in micList: # We need to make this dict because there is no ID in the # coord.star file coordDict = {} for coord in self.coordDict[mic.getObjId()]: coordDict[self._getPos(coord)] = coord _, partStarFn = self._getStarFiles(mic) for row in md.iterRows(self._getPath(partStarFn)): pos = (row.getValue(md.RLN_IMAGE_COORD_X), row.getValue(md.RLN_IMAGE_COORD_Y)) coord = coordDict.get(pos, None) if coord is not None: # scale the coordinates according to particles dimension. coord.scale(self.getBoxScale()) p.copyObjId(coord) idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME)) p.setLocation(idx, self._getPath(fn[2:])) p.setCoordinate(coord) p.setMicId(mic.getObjId()) p.setCTF(mic.getCTF()) outputParts.append(p) # Release the list of coordinates for this micrograph since it # will not be longer needed del self.coordDict[mic.getObjId()]
def _loadClassesInfo(self): """ Read some information about the produced Relion 3D classes from the *model.star file. """ self._classesInfo = {} # store classes info, indexed by class id mdModel = self._getFileName('outputModel', lev=self._level) modelStar = md.MetaData('model_classes@' + mdModel) for classNumber, row in enumerate(md.iterRows(modelStar)): index, fn = relionToLocation(row.getValue('rlnReferenceImage')) # Store info indexed by id, we need to store the row.clone() since # the same reference is used for iteration self._classesInfo[classNumber + 1] = (index, fn, row.clone())
def convertInputStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. Params: particlesId: use this parameter just to force redo of convert if the input particles are changed. """ imgSet = self._getInputParticles() imgStar = self._getFileName('movie_particles') imgStarTmp = self._getTmpPath('movie_particles.star') self.info("Converting set from '%s' into '%s'" % (imgSet.getFileName(), imgStarTmp)) writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(), alignType=imgSet.getAlignment(), extraLabels=MOVIE_EXTRA_LABELS) mdImg = md.MetaData(imgStarTmp) # replace mdColumn from *.stk to *.mrcs as Relion2 requires if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME from convert import relionToLocation, locationToRelion for objId in mdImg: index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId)) if not imgPath.endswith('mrcs'): newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs') newPath = self._getTmpPath(newName) newLoc = locationToRelion(index, newPath) if not exists(newPath): pwutils.createLink(imgPath, newPath) mdImg.setValue(mdColumn, newLoc, objId) mdImg.write(imgStar)
def organizeDataStep(self): from pyworkflow.utils import moveFile import pyworkflow.em.metadata as md from convert import relionToLocation, locationToRelion # moving shiny.star form project base path to the current protocol extra path. shinyStar = "shiny.star" pathFixedShiny = self._getExtraPath(shinyStar) if exists(shinyStar): moveFile(shinyStar, pathFixedShiny) mdShiny = md.MetaData(pathFixedShiny) oldImgPath = "" for objId in mdShiny: index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = self._getExtraPath(os.path.basename(imgPath)) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldImgPath != imgPath and exists(imgPath): moveFile(imgPath, newPath) oldImgPath = imgPath mdShiny.write(pathFixedShiny)
def organizeDataStep(self): from pyworkflow.utils import moveFile import pyworkflow.em.metadata as md from convert import relionToLocation, locationToRelion # moving shiny.star form project base path to the current protocol extra path. shinyStar = "shiny.star" pathFixedShiny = self._getExtraPath(shinyStar) if exists(shinyStar): moveFile(shinyStar, pathFixedShiny) mdShiny = md.MetaData(pathFixedShiny) oldImgPath = "" for objId in mdShiny: index, imgPath = relionToLocation( mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = self._getExtraPath(os.path.basename(imgPath)) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldImgPath != imgPath and exists(imgPath): moveFile(imgPath, newPath) oldImgPath = imgPath mdShiny.write(pathFixedShiny)
def _updateItem(self, item, row): newFn = row.getValue(md.RLN_IMAGE_NAME) newLoc = relionToLocation(newFn) item.setLocation(newLoc)
def readPartsFromMics(self, micList, outputParts): """ Read the particles extract for the given list of micrographs and update the outputParts set with new items. """ p = em.Particle() # Let's create a dict with the names of the mic in Relion star files # and also create a set with all star files to iterate them once starSet = set() micPathDict = {} for mic in micList: starSet.add(self._micCoordStarDict[mic.getObjId()]) micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc') micFile = os.path.join('extra', micName) micPathDict[micFile] = mic prevMicFile = None mic = None for partStarFn in starSet: for row in md.iterRows(self._getPath(partStarFn), sortByLabel=md.RLN_MICROGRAPH_NAME): micFile = row.getValue(md.RLN_MICROGRAPH_NAME) if micFile != prevMicFile: # Load some stuff when new mic if prevMicFile is not None: # cleanup previous mic # Release the list of coordinates for this micrograph # since it will not be longer needed del self.coordDict[mic.getObjId()] mic = micPathDict[micFile] coordDict = {self._getPos(coord): coord for coord in self.coordDict[mic.getObjId()]} posSet = set() # Set of (x, y) pairs to avoid duplicates prevMicFile = micFile pos = (row.getValue(md.RLN_IMAGE_COORD_X), row.getValue(md.RLN_IMAGE_COORD_Y)) if pos in posSet: print("Duplicate coordinate at: %s, IGNORED. " % str(pos)) coord = None else: coord = coordDict.get(pos, None) if coord is not None: # scale the coordinates according to particles dimension. coord.scale(self.getBoxScale()) p.copyObjId(coord) idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME)) p.setLocation(idx, self._getPath(fn[2:])) p.setCoordinate(coord) p.setMicId(mic.getObjId()) p.setCTF(mic.getCTF()) outputParts.append(p) posSet.add(pos) # Clean up the last mic if necessary if mic is not None: del self.coordDict[mic.getObjId()]
def readPartsFromMics(self, micList, outputParts): """ Read the particles extract for the given list of micrographs and update the outputParts set with new items. """ p = em.Particle() # Let's create a dict with the names of the mic in Relion star files # and also create a set with all star files to iterate them once starSet = set() micPathDict = {} for mic in micList: starSet.add(self._micCoordStarDict[mic.getObjId()]) micName = pwutils.replaceBaseExt(mic.getFileName(), 'mrc') micFile = os.path.join('extra', micName) micPathDict[micFile] = mic prevMicFile = None mic = None for partStarFn in starSet: for row in md.iterRows(self._getPath(partStarFn), sortByLabel=md.RLN_MICROGRAPH_NAME): micFile = row.getValue(md.RLN_MICROGRAPH_NAME) if micFile != prevMicFile: # Load some stuff when new mic if prevMicFile is not None: # cleanup previous mic # Release the list of coordinates for this micrograph # since it will not be longer needed del self.coordDict[mic.getObjId()] mic = micPathDict[micFile] coordDict = {self._getPos(coord): coord for coord in self.coordDict[mic.getObjId()]} posSet = set() # Set of (x, y) pairs to avoid duplicates prevMicFile = micFile pos = (row.getValue(md.RLN_IMAGE_COORD_X), row.getValue(md.RLN_IMAGE_COORD_Y)) if pos in posSet: print("Duplicate coordinate at: %s, IGNORED. " % str(pos)) coord = None else: coord = coordDict.get(pos, None) if coord is not None: # scale the coordinates according to particles dimension. coord.scale(self.getBoxScale()) p.copyObjId(coord) idx, fn = relionToLocation(row.getValue(md.RLN_IMAGE_NAME)) p.setLocation(idx, self._getPath(fn[2:])) p.setCoordinate(coord) p.setMicId(mic.getObjId()) p.setCTF(mic.getCTF()) outputParts.append(p) posSet.add(pos) # Clean up the last mic if necessary if mic is not None: del self.coordDict[mic.getObjId()]
def _updateItem(self, item, row): newFn = row.getValue(md.RLN_IMAGE_NAME) newLoc = relionToLocation(newFn) item.setLocation(newLoc)
def _updateItem(self, particle, row): newLoc = convert.relionToLocation(row.getValue('rlnImageName')) particle.setLocation(newLoc)
def _plotClassDistribution(self, paramName=None): labels = ["rlnClassDistribution", "rlnAccuracyRotations", "rlnAccuracyTranslations"] iterations = range(self.firstIter, self.lastIter + 1) classInfo = {} for it in iterations: modelStar = self.protocol._getFileName('model', iter=it) for row in md.iterRows('%s@%s' % ('model_classes', modelStar)): i, fn = relionToLocation(row.getValue('rlnReferenceImage')) if i == em.NO_INDEX: # the case for 3D classes # NOTE: Since there is not an proper ID value in # the clases metadata, we are assuming that class X # has a filename *_classXXX.mrc (as it is in Relion) # and we take the ID from there index = int(fn[-7:-4]) else: index = i if index not in classInfo: classInfo[index] = {} for l in labels: classInfo[index][l] = [] for l in labels: classInfo[index][l].append(row.getValue(l)) xplotter = VolSelPlotter() xplotter.createSubPlot("Classes distribution over iterations", "Iterations", "Classes Distribution") # Empty list for each iteration iters = [[]] * len(iterations) l = labels[0] for index in sorted(classInfo.keys()): for it, value in enumerate(classInfo[index][l]): iters[it].append(value) ax = xplotter.getLastSubPlot() n = len(iterations) ind = range(n) bottomValues = [0] * n width = 0.45 # the width of the bars: can also be len(x) sequence def get_cmap(N): import matplotlib.cm as cmx import matplotlib.colors as colors """Returns a function that maps each index in 0, 1, ... N-1 to a distinct RGB color.""" color_norm = colors.Normalize(vmin=0, vmax=N)#-1) scalar_map = cmx.ScalarMappable(norm=color_norm, cmap='hsv') def map_index_to_rgb_color(index): return scalar_map.to_rgba(index) return map_index_to_rgb_color cmap = get_cmap(len(classInfo)) for classId in sorted(classInfo.keys()): values = classInfo[classId][l] ax.bar(ind, values, width, label='class %s' % classId, bottom=bottomValues, color=cmap(classId)) bottomValues = [a+b for a, b in zip(bottomValues, values)] ax.get_xaxis().set_ticks([i + 0.25 for i in ind]) ax.get_xaxis().set_ticklabels([str(i) for i in ind]) ax.legend(loc='upper left', fontsize='xx-small') return [xplotter]