def _processMovie(self, movieId, movieName, movieFolder): """call program here""" # if not mrc convert format to mrc # special case is mrc but ends in mrcs inMovieName = os.path.join(movieFolder, movieName) if movieName.endswith(".mrc"): movieNameAux = inMovieName elif movieName.endswith(".mrcs"): movieNameAux = pwutils.replaceExt(inMovieName, "mrc") createLink(inMovieName, movieNameAux) movieNameAux = pwutils.replaceExt(movieName, "mrc") else: micFnMrc = pwutils.replaceExt(inMovieName, "mrc") ImageHandler().convert(inMovieName, micFnMrc, DT_FLOAT) movieNameAux = pwutils.replaceExt(movieName, "mrc") # get number of frames if self.alignFrameRange == -1: numberOfFramesPerMovie = self.inputMovies.get().getDimensions()[3] else: numberOfFramesPerMovie = self.alignFrameRange.get() doApplyDoseFilter = self.doApplyDoseFilter.get() exposurePerFrame = self.exposurePerFrame.get() self._argsUnblur( movieNameAux, movieFolder, movieId, numberOfFramesPerMovie, doApplyDoseFilter, exposurePerFrame ) try: self.runJob(self._program, self._args, cwd=movieFolder) except: print("ERROR: Movie %s failed\n" % movieName) logFile = self._getLogFile(movieId)
def _visualize(self, obj, **args): cls = type(obj) if issubclass(cls, Volume) or issubclass(cls, Image): fn = obj.getFileName() if pwutils.getExt(fn) in ['.stk', '.vol', '.xmp']: # eman2 expects spider files with .spi extension only pwutils.createLink(fn, pwutils.replaceExt(fn, 'spi')) fn = pwutils.replaceExt(fn, 'spi') view = CommandView("%s %s &" % (Plugin.getProgram('e2display.py'), fn), env=Plugin.getEnviron()) return [view] elif isinstance(obj, EmanProtBoxing): coords = obj.getCoords() if coords: return DataViewer._visualize(self, obj.outputCoordinates) elif isinstance(obj, EmanProtInitModel): obj = obj.outputVolumes fn = obj.getFileName() labels = 'id enabled comment _filename ' objCommands = "'%s' '%s' '%s'" % (OBJCMD_CLASSAVG_PROJS, OBJCMD_PROJS, OBJCMD_INITVOL) self._views.append( ObjectView(self._project, obj.strId(), fn, viewParams={ showj.MODE: showj.MODE_MD, showj.VISIBLE: labels, showj.RENDER: '_filename', showj.OBJCMDS: objCommands })) return self._views elif isinstance(obj, EmanProtInitModelSGD): obj = obj.outputVolumes fn = obj.getFileName() labels = 'id enabled comment _filename ' objCommands = "'%s'" % OBJCMD_CLASSAVG_PROJS self._views.append( ObjectView(self._project, obj.strId(), fn, viewParams={ showj.MODE: showj.MODE_MD, showj.VISIBLE: labels, showj.RENDER: '_filename', showj.OBJCMDS: objCommands })) return self._views
def getFinalCorrectionImagePath(self, correctionImage): """ Returns the final path to the correction image (converted or not) or and exception correctionImage does not exists""" # Return if the correctionImage is None or the empty string if not correctionImage: return None elif not os.path.exists(correctionImage): raise Exception( "Correction image is set but not present in the " "file system. Either clean the value in the import " "protocol or copy the file to %s." % correctionImage) # Do we need to convert? convertTo = self._getConvertExtension(correctionImage) if convertTo is not None: # Get the base name fileName = basename(correctionImage) # Replace extension fileName = pwutils.replaceExt(fileName, convertTo) # "Place" it at extra folder. return self._getExtraPath(fileName) else: return correctionImage
def _showOneColorslice(self, param=None): if (exists(self.protocol._getExtraPath("mgresolution.mrc"))): imageFile = self.protocol._getExtraPath("mgresolution.mrc") else: imageFile = self.protocol._getExtraPath(OUTPUT_RESOLUTION_FILE) if not os.path.exists(imageFile): imageFile = replaceExt(imageFile, 'mrc') imgData, min_Res, max_Res, voldim = self.getImgData(imageFile) xplotter = XmippPlotter(x=1, y=1, mainTitle="Local Resolution Slices " "along %s-axis." % self._getAxis()) sliceNumber = self.sliceNumber.get() if sliceNumber < 0: sliceNumber = int(voldim[0] / 2) else: sliceNumber -= 1 # sliceNumber has no sense to start in zero a = xplotter.createSubPlot("Slice %s" % (sliceNumber + 1), '', '') matrix = self.getSliceImage(imgData, sliceNumber, self._getAxis()) plot = xplotter.plotMatrix(a, matrix, self.lowest.get(), self.highest.get(), cmap=self.getColorMap(), interpolation="nearest") xplotter.getColorBar(plot) return [xplotter]
def __convertCorrectionImage(self, correctionImage): """ Will convert a gain or dark file to a compatible one and return the final file name. Otherwise, will return same passed parameter""" # Get final correction image file finalName = self.getFinalCorrectionImagePath(correctionImage) # If correctionImage is None, finalName will be None if finalName is None: return None elif not os.path.exists(finalName): # Conversion never happened... print('converting %s to %s' % (correctionImage, finalName)) if correctionImage.endswith('.gain'): # treat as tif file fnBase = basename(correctionImage) tmpLink = self._getTmpPath(pwutils.replaceExt(fnBase, 'tif')) pwutils.createAbsLink(correctionImage, tmpLink) correctionImage = tmpLink emlib.image.ImageHandler().convert(correctionImage, finalName) # return final name return os.path.abspath(finalName)
def appendMicsFromTomogram(self, output, tomo): self.info("Creating micrographs for %s" % tomo.getFileName()) # Load the tomogram ih = ImageHandler() img = ih.read(tomo.getFileName()) data = img.getData() # For each slice for index in range(0, len(data), self.slicesGap.get()): self.debug("Creating micrograph for slice %s" % index) micName = tomoSliceToMicName(tomo, index) outputMicName = self._getExtraPath(micName) outputMicName = replaceExt(outputMicName, "mrc") slice = data[index] micImg = ImageHandler() micImg._img.setData(slice) micImg.write(micImg._img, outputMicName) # Create the micrograph metadata object newMic = Micrograph() newMic.setFileName(outputMicName) newMic.setMicName(micName) newMic.setSamplingRate(tomo.getSamplingRate()) # Append it output.append(newMic)
def _processMovie(self, movieId, movieName, movieFolder): # if not mrc convert format to mrc # special case is mrc but ends in mrcs moviePath = os.path.join(movieFolder, movieName) movieNameMrc = pwutils.replaceExt(movieName, "mrc") moviePathMrc = pwutils.replaceExt(moviePath, "mrc") ih = ImageHandler() if movieName.endswith('.mrc'): pass # Do nothing elif movieName.endswith('.mrcs'): # Create a link to the mrc or mrcs file but with .mrc extension createLink(moviePath, moviePathMrc) else: # Convert to mrc if the movie is in other format ih.convert(moviePath, moviePathMrc, DT_FLOAT) _, _, z, n = ih.getDimensions(moviePathMrc) numberOfFrames = max(z, n) # Deal with mrc ambiguity # Write dummy auxiliary shift file. # TODO: this should be done properly when we define # how to transfer between movies shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId)) f = open(shiftFnName,'w') shift = ("0 " * numberOfFrames + "\n" ) * 2 f.write(shift) f.close() if self.alignFrameRange != -1: if self.alignFrameRange > numberOfFrames: raise Exception('Frame number (%d) is greater than ' 'the total frames of the movie (%d)' % (numberOfFrames, self.alignFrameRange)) numberOfFrames = self.alignFrameRange.get() self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames) try: self.runJob(self._program, self._args, cwd=movieFolder) except: print("ERROR: Movie %s failed\n"%movieName) logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder): # if not mrc convert format to mrc # special case is mrc but ends in mrcs moviePath = os.path.join(movieFolder, movieName) movieNameMrc = pwutils.replaceExt(movieName, "mrc") moviePathMrc = pwutils.replaceExt(moviePath, "mrc") ih = ImageHandler() if movieName.endswith('.mrc'): pass # Do nothing elif movieName.endswith('.mrcs'): # Create a link to the mrc or mrcs file but with .mrc extension createLink(moviePath, moviePathMrc) else: # Convert to mrc if the movie is in other format ih.convert(moviePath, moviePathMrc, DT_FLOAT) _, _, z, n = ih.getDimensions(moviePathMrc) numberOfFrames = max(z, n) # Deal with mrc ambiguity # Write dummy auxiliary shift file. # TODO: this should be done properly when we define # how to transfer between movies shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId)) f = open(shiftFnName, 'w') shift = ("0 " * numberOfFrames + "\n") * 2 f.write(shift) f.close() if self.alignFrameRange != -1: if self.alignFrameRange > numberOfFrames: raise Exception('Frame number (%d) is greater than ' 'the total frames of the movie (%d)' % (numberOfFrames, self.alignFrameRange)) numberOfFrames = self.alignFrameRange.get() self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames) try: self.runJob(self._program, self._args, cwd=movieFolder) except: print("ERROR: Movie %s failed\n" % movieName) logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder): """call program here""" # if not mrc convert format to mrc # special case is mrc but ends in mrcs inMovieName = os.path.join(movieFolder, movieName) if movieName.endswith(".mrc"): movieNameAux = inMovieName elif movieName.endswith(".mrcs"): movieNameAux = pwutils.replaceExt(inMovieName, "mrc") createLink(inMovieName, movieNameAux) movieNameAux = pwutils.replaceExt(movieName, "mrc") else: micFnMrc = pwutils.replaceExt(inMovieName, "mrc") ImageHandler().convert(inMovieName, micFnMrc, DT_FLOAT) movieNameAux = pwutils.replaceExt(movieName, "mrc") # get number of frames if self.alignFrameRange == -1: numberOfFramesPerMovie = self.inputMovies.get().getDimensions()[3] else: numberOfFramesPerMovie = self.alignFrameRange.get() # write dummy auxiliary shift file. # TODO: this should be done properly when we define how to transfer shift # between movies shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId)) f = open(shiftFnName, "w") shift = ("0 " * numberOfFramesPerMovie + "\n") * 2 f.write(shift) f.close() doApplyDoseFilter = self.doApplyDoseFilter.get() exposurePerFrame = self.exposurePerFrame.get() self._argsSummovie( movieNameAux, movieFolder, movieId, numberOfFramesPerMovie, doApplyDoseFilter, exposurePerFrame ) try: self.runJob(self._program, self._args, cwd=movieFolder) except: print("ERROR: Movie %s failed\n" % movieName) logFile = self._getLogFile(movieId)
def _getOutputMicrograph(self, mic): """ Return the name of the output micrograph, given the input Micrograph object. """ fn = mic.getFileName() extFn = getExt(fn) if extFn != ".mrc": fn = replaceExt(fn, "mrc") fnOut = self._getExtraPath(basename(fn)) return fnOut
def _processMovie(self, movieId, movieName, movieFolder): # if not mrc convert format to mrc # special case is mrc but ends in mrcs moviePath = os.path.join(movieFolder, movieName) movieNameMrc = pwutils.replaceExt(movieName, "mrc") moviePathMrc = pwutils.replaceExt(moviePath, "mrc") ih = ImageHandler() if movieName.endswith('.mrc'): pass # Do nothing elif movieName.endswith('.mrcs'): # Create a link to the mrc or mrcs file but with .mrc extension createLink(moviePath, moviePathMrc) else: # Convert to mrc if the movie is in other format ih.convert(moviePath, moviePathMrc, DT_FLOAT) _, _, z, n = ih.getDimensions(moviePathMrc) numberOfFrames = max(z, n) # Deal with mrc ambiguity if self.alignFrameRange != -1: if self.alignFrameRange > numberOfFrames: raise Exception('Frame number (%d) is greater than ' 'the total frames of the movie (%d)' % (numberOfFrames, self.alignFrameRange)) numberOfFrames = self.alignFrameRange.get() self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames) try: self.runJob(self._program, self._args, cwd=movieFolder) except: print("ERROR: Movie %s failed\n" % movieName) logFile = self._getLogFile(movieId)
def _pickMicrographStep(self, mics, args): """ Main func that runs the picking job. :param mics: micrograph list :param args: programs args """ for mic in mics: micName = mic.getFileName() outMic = os.path.join(self._getTmpPath(), pwutils.replaceBaseExt(micName, 'mrc')) ctf = self.ctfDict[mic.getMicName()] args.update({'micName': outMic, 'logFn': self._getLogFn(mic), 'outStack': self._getStackFn(mic), 'phaseShift': ctf.getPhaseShift() or 0.0, 'defocusU': ctf.getDefocusU(), 'defocusV': ctf.getDefocusV(), 'defocusAngle': ctf.getDefocusAngle() }) if self.pickType == 1: args.update({ 'refsFn': self._getExtraPath('references.mrc'), 'useRadAvg': 'YES' if self.useRadAvg else 'NO', 'rotateRef': self.rotateRef.get(), }) argsStr = self._getArgsStr() cmdArgs = argsStr % args try: self.runJob(self._getProgram(), cmdArgs, env=Plugin.getEnviron()) # Move output coords from tmp to extra pltFn = pwutils.replaceExt(self._getStackFn(mic), 'plt') pwutils.moveFile(pltFn, self._getPltFn(mic)) # Clean tmp folder pwutils.cleanPath(outMic) pwutils.cleanPath(self._getLogFn(mic)) pwutils.cleanPath(self._getStackFn(mic)) except Exception as e: self.error("ERROR: Picking has failed for %s. %s" % ( outMic, self._getErrorFromPickerTxt(mic, e)))
def _showVolumeColorSlices(self, param=None): if (exists(self.protocol._getExtraPath("mgresolution.mrc"))): imageFile = self.protocol._getExtraPath("mgresolution.mrc") else: imageFile = self.protocol._getExtraPath(OUTPUT_RESOLUTION_FILE) if not os.path.exists(imageFile): imageFile = replaceExt(imageFile, 'mrc') imgData, min_Res, max_Res, voldim = self.getImgData(imageFile) xplotter = XmippPlotter(x=2, y=2, mainTitle="Local Resolution Slices " "along %s-axis." % self._getAxis()) # The slices to be shown are close to the center. Volume size is divided in # 9 segments, the fouth central ones are selected i.e. 3,4,5,6 for i in range(3, 7): sliceNumber = self.getSlice(i, imgData) a = xplotter.createSubPlot("Slice %s" % (sliceNumber + 1), '', '') matrix = self.getSliceImage(imgData, sliceNumber, self._getAxis()) plot = xplotter.plotMatrix(a, matrix, self.lowest.get(), self.highest.get(), cmap=self.getColorMap(), interpolation="nearest") xplotter.getColorBar(plot) return [xplotter]
def launchTest(self, fileKey, mList, alignType=None, **kwargs): """ Helper function to launch similar alignment tests given the EMX transformation matrix. Params: fileKey: the file where to grab the input stack images. mList: the matrix list of transformations (should be the same length of the stack of images) """ print("\n") print("*" * 80) print("* Launching test: ", fileKey) print("*" * 80) is2D = alignType == ALIGN_2D if fileKey == 'alignShiftRotExp': # relion requires mrcs stacks origFn = self.dataset.getFile(fileKey) stackFn = replaceExt(origFn, ".mrcs") createLink(origFn, stackFn) else: stackFn = self.dataset.getFile(fileKey) partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite") mdFn = self.getOutputPath(fileKey + "_particles.star") partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite") if self.IS_ALIGNMENT: outputFn = self.getOutputPath(fileKey + "_output.mrcs") outputFnRelion = self.getOutputPath(fileKey + "_output") goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs') else: outputFn = self.getOutputPath(fileKey + "_output.vol") goldFn = self.dataset.getFile("reconstruction/gold/" + fileKey + '_Gold_rln_output.vol') if PRINT_FILES: print("BINARY DATA: ", stackFn) print("SET1: ", partFn1) print(" MD: ", mdFn) print("SET2: ", partFn2) print("OUTPUT: ", outputFn) print("GOLD: ", goldFn) if alignType == ALIGN_2D or alignType == ALIGN_PROJ: partSet = SetOfParticles(filename=partFn1) else: partSet = SetOfVolumes(filename=partFn1) partSet.setAlignment(alignType) acq = Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000) og = OpticsGroups.create(rlnMtfFileName="mtfFile1.star", rlnImageSize=128) partSet.setSamplingRate(1.0) partSet.setAcquisition(acq) og.toImages(partSet) # Populate the SetOfParticles with images # taken from images.mrc file # and setting the previous alignment parameters aList = [np.array(m) for m in mList] for i, a in enumerate(aList): p = Particle() p.setLocation(i + 1, stackFn) p.setTransform(Transform(a)) partSet.append(p) # Write out the .sqlite file and check that are correctly aligned print("Partset", partFn1) partSet.printAll() partSet.write() # Convert to a Xmipp metadata and also check that the images are # aligned correctly if alignType == ALIGN_2D or alignType == ALIGN_PROJ: starWriter = convert.createWriter() starWriter.writeSetOfParticles(partSet, mdFn, alignType=alignType) partSet2 = SetOfParticles(filename=partFn2) else: convert.writeSetOfVolumes(partSet, mdFn, alignType=alignType) partSet2 = SetOfVolumes(filename=partFn2) # Let's create now another SetOfImages reading back the written # Xmipp metadata and check one more time. partSet2.copyInfo(partSet) if alignType == ALIGN_2D or alignType == ALIGN_PROJ: convert.readSetOfParticles(mdFn, partSet2, alignType=alignType) else: convert.readSetOfParticles(mdFn, partSet2, rowToFunc=convert.rowToVolume, alignType=alignType) partSet2.write() if PRINT_MATRIX: for i, img in enumerate(partSet2): m1 = aList[i] m2 = img.getTransform().getMatrix() print("-" * 5) print(img.getFileName(), img.getIndex()) print('m1:\n', m1, convert.geometryFromMatrix(m1, False)) print('m2:\n', m2, convert.geometryFromMatrix(m2, False)) self.assertTrue(np.allclose(m1, m2, rtol=1e-2)) # Launch apply transformation and check result images runRelionProgram(self.CMD % locals()) if SHOW_IMAGES: runRelionProgram('scipion show %(outputFn)s' % locals()) if os.path.exists(goldFn): self.assertTrue( ImageHandler().compareData(goldFn, outputFn, tolerance=0.001), "Different data files:\n>%s\n<%s" % (goldFn, outputFn)) if CLEAN_IMAGES: cleanPath(outputFn)
def _getMovieFn(self, movie): movieFn = movie.getFileName() if movieFn.endswith("tiff"): return pwutils.replaceExt(movieFn, "tif") else: return movieFn
def _getOutputVolFn(self, fn): return replaceExt(fn, '_origSize.mrc')
def _getMovieFn(self, movie): movieFn = movie.getFileName() if movieFn.endswith("mrcs"): return pwutils.replaceExt(movieFn, self.CONVERT_TO_MRC) else: return movieFn
def generate(self, finished): project = self.protocol.getProject() # Project Properties Section # Do not delete this variables. We are using them # in an eval command self.projectName = project.getShortName() startTime = pwutils.dateStr(project.getCreationTime(), secs=True), tnow = datetime.now() _now = project.getCreationTime() dateStr = pwutils.prettyTime(dt=tnow, secs=True), projectDuration = pwutils.prettyDelta(tnow - project.getCreationTime()), projectStatus = "FINISHED" if finished else "RUNNING", scipionVersion = os.environ['SCIPION_VERSION'], # create structure with information related with the properties section # the primmary key is time, projectionName, section # if we insert two points with these three values idential # the second updates the first pointsDict = {} # dictionary for data points pointsDict['measurement'] = self.projectName tags = {} tags['section'] = 'properties' pointsDict['tags'] = tags fields = {} firstTime = self.confParser.getint("project", "properties") fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2} fieldNames = { 'dateStr': "<b>Last Update</b>", 'projectDuration': '<b>Duration</b>', 'projectStatus': '<b>Status</b>' } if firstTime: fieldKeys['startTime'] = 5 fieldKeys['scipionVersion'] = 1 fieldNames['startTime'] = '<b>Start Time</b>' fieldNames['scipionVersion'] = '<b>Scipion Version</b>' for metric, delta in fieldKeys.items(): localNow = _now + timedelta(seconds=delta) pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ') fields['metric'] = fieldNames[metric] # str is need because all values must have the same type fields['valueStr'] = str(eval(metric)[0]) pointsDict['fields'] = fields self.client.write_points([pointsDict]) self.confParser.set("project", "properties", "0") with open(self.confFileName, 'w') as confFile: self.confParser.write(confFile) # acquisition section self.provider.refreshObjects() pointsDict = {} # dictionary for data points pointsDict['measurement'] = self.projectName tags = {} tags['section'] = 'acquisition' pointsDict['tags'] = tags fields = {} firstTime = self.confParser.getint("project", "acquisition") if firstTime: delta = 0 for metricName, value in self.provider.acquisition: localNow = _now + timedelta(seconds=delta) pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ') fields['metric'] = "<b>" + metricName + "</b>" fields['valueNum'] = float(value) pointsDict['fields'] = fields self.client.write_points([pointsDict]) delta = delta - 1 # update first time only if some date has been read. # do not place this upside the loop self.confParser.set("project", "acquisition", "0") with open(self.confFileName, 'w') as confFile: self.confParser.write(confFile) # send summary section pointsDict = {} # dictionary for data points pointsDict['measurement'] = self.projectName tags = {} tags['section'] = 'summary' pointsDict['tags'] = tags localNow = _now for obj in self.provider.getObjects(): fields = {} localNow = localNow + timedelta(seconds=-1) pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ') # If it's a protocol isProtocol = True if obj.name else False if isProtocol: fields['protocolName'] = '<b>' + obj.name + '</b>' fields['output'] = "" fields['size'] = "" else: fields['protocolName'] = "" fields['output'] = obj.output fields['size'] = str(obj.outSize) pointsDict['fields'] = fields self.client.write_points([pointsDict]) # Ctf monitor chart data last_id = self.confParser.getint("ctf", "lastId") listDictionaryCTF = {} if self.ctfMonitor is None else \ self.ctfMonitor.getData(lastId=last_id) if listDictionaryCTF: pointsDict = {} # dictionary for data points pointsDict['measurement'] = self.projectName tags = {} tags['section'] = 'ctf' for ctf in listDictionaryCTF: fields = {} for key in ctf: if key == 'timestamp': pointsDict['time'] = ctf['timestamp'] # id must be a tag since tw CTF may have the same timestamp # and time + tags must be unique elif key == 'id': tags['id'] = ctf['id'] # elif key == 'ctfID': # fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>" elif key == 'shiftPlotPath': temp = os.path.join(self.apacheImgDir, self.projectName, basename(ctf[key])) popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \ (temp, temp, basename(ctf[key])) fields[key] = popUpStr fields[key + 'Local'] = ctf[key] elif key == 'psdPath': # convert to pǹg baseName = basename(ctf[key]) baseNamePng = pwutils.replaceBaseExt(ctf[key], "png") temp = os.path.join(self.apacheImgDir, self.projectName, baseName) temp = pwutils.replaceExt(temp, "png") popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \ (temp, temp, baseNamePng) fields[key] = popUpStr fields[key + 'Local'] = ctf[key] fields[key + 'LocalPng'] = pwutils.replaceExt( ctf[key], "png") elif key == 'micPath': # convert to pǹg baseName = basename(ctf[key]) baseNamePng = pwutils.replaceBaseExt(ctf[key], "png") temp = os.path.join(self.apacheImgDir, self.projectName, baseName) temp = pwutils.replaceExt(temp, "png") popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \ (temp, temp, baseNamePng) fields[key] = popUpStr fields[key + 'Local'] = ctf[key] fields[key + 'LocalPng'] = pwutils.replaceExt( ctf[key], "png") else: fields[key] = ctf[key] # while be use to control image creation fields["transferImage"] = False pointsDict['fields'] = fields pointsDict['tags'] = tags self.client.write_points([pointsDict]) last_id += 1 self.confParser.set("ctf", "lastId", str(last_id)) with open(self.confFileName, 'w') as confFile: self.confParser.write(confFile) # GAIN Section last_id = self.confParser.getint("gain", "lastId") listDictionaryGain = {} if self.movieGainMonitor is None else \ self.movieGainMonitor.getData(lastId=last_id) if listDictionaryGain: pointsDict = {} # dictionary for data points pointsDict['measurement'] = self.projectName tags = {} tags['section'] = 'gain' pointsDict['tags'] = tags counter = 1 tnow = datetime.now() for gain in listDictionaryGain: fields = {} for key in gain: fields[key] = gain[key] pointsDict['fields'] = fields # gain has no time information so I just # put current time localNow = tnow + timedelta(seconds=counter) pointsDict[ 'time'] = localNow # .strftime('%Y-%m-%dT%H:%M:%SZ') self.client.write_points([pointsDict]) last_id += 1 self.confParser.set("gain", "lastId", str(last_id)) with open(self.confFileName, 'w') as confFile: self.confParser.write(confFile) # SYSTEM data last_id = self.confParser.getint("system", "lastId") listDictionarySystem = {} if self.sysMonitor is None else \ self.sysMonitor.getData(lastId=last_id) if listDictionarySystem: pointsDict = {} # dictionary for data points pointsDict['measurement'] = self.projectName tags = {} tags['section'] = 'system' pointsDict['tags'] = tags for system in listDictionarySystem: fields = {} for key in system: if key == 'timestamp': pointsDict['time'] = system['timestamp'] elif key == 'id': continue else: fields[key] = system[key] pointsDict['fields'] = fields self.client.write_points([pointsDict]) last_id += 1 self.confParser.set("system", "lastId", str(last_id)) with open(self.confFileName, 'w') as confFile: self.confParser.write(confFile) self.transferFiles() return last_id # reportFinished
def getFinalMaskFileName(inMask): if getExt(inMask.getFileName()) in COMP_EXT_MASK_LIST: return abspath(inMask.getFileName()) else: return abspath(replaceExt(inMask.getFileName(), '.mrc'))
def getThumbnailFn(self, inputFn): """ Returns the default name for a thumbnail image""" return pwutils.replaceExt(inputFn, "thumb.png")
def getThumbnailFn(inputFn): """Replace the extension in inputFn with thumb.png""" return pwutils.replaceExt(inputFn, "thumb.png")
def processMovieStep(self, movieDict, hasAlignment): movie = emobj.Movie() movie.setAcquisition(emobj.Acquisition()) if hasAlignment: movie.setAlignment(emobj.MovieAlignment()) movie.setAttributesFromDict(movieDict, setBasic=True, ignoreMissing=True) movieFolder = self._getOutputMovieFolder(movie) movieFn = movie.getFileName() movieName = basename(movieFn) movieDoneFn = self._getMovieDone(movie) if self.isContinued() and os.path.exists(movieDoneFn): self.info("Skipping movie: %s, seems to be done" % movieFn) return # Clean old finished files pwutils.cleanPath(movieDoneFn) if self._filterMovie(movie): pwutils.makePath(movieFolder) pwutils.createLink(movieFn, join(movieFolder, movieName)) if movieName.endswith('bz2'): newMovieName = movieName.replace('.bz2', '') # We assume that if compressed the name ends with .mrc.bz2 if not exists(newMovieName): self.runJob('bzip2', '-d -f %s' % movieName, cwd=movieFolder) elif movieName.endswith('tbz'): newMovieName = movieName.replace('.tbz', '.mrc') # We assume that if compressed the name ends with .tbz if not exists(newMovieName): self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder) elif movieName.endswith('.txt'): # Support a list of frame as a simple .txt file containing # all the frames in a raw list, we could use a xmd as well, # but a plain text was choose to simply its generation movieTxt = os.path.join(movieFolder, movieName) with open(movieTxt) as f: movieOrigin = os.path.basename(os.readlink(movieFn)) newMovieName = movieName.replace('.txt', '.mrcs') ih = emlib.image.ImageHandler() for i, line in enumerate(f): if line.strip(): inputFrame = os.path.join(movieOrigin, line.strip()) ih.convert( inputFrame, (i + 1, os.path.join(movieFolder, newMovieName))) else: newMovieName = movieName convertExt = self._getConvertExtension(newMovieName) correctGain = self._doCorrectGain() if convertExt or correctGain: inputMovieFn = os.path.join(movieFolder, newMovieName) if inputMovieFn.endswith('.em'): inputMovieFn += ":ems" if convertExt: newMovieName = pwutils.replaceExt(newMovieName, convertExt) else: newMovieName = '%s_corrected.%s' % os.path.splitext( newMovieName) outputMovieFn = os.path.join(movieFolder, newMovieName) # If the protocols wants Scipion to apply the gain, then # there is no reason to convert, since we can produce the # output in the format expected by the program. In some cases, # the alignment programs can directly deal with gain and dark # correction images, so we don't need to apply it if self._doCorrectGain(): self.info("Correcting gain and dark '%s' -> '%s'" % (inputMovieFn, outputMovieFn)) gain, dark = self.getGainAndDark() self.correctGain(inputMovieFn, outputMovieFn, gainFn=gain, darkFn=dark) else: self.info("Converting movie '%s' -> '%s'" % (inputMovieFn, outputMovieFn)) emlib.image.ImageHandler().convertStack( inputMovieFn, outputMovieFn) # Just store the original name in case it is needed in _processMovie movie._originalFileName = pwobj.String(objDoStore=False) movie._originalFileName.set(movie.getFileName()) # Now set the new filename (either linked or converted) movie.setFileName(os.path.join(movieFolder, newMovieName)) self.info("Processing movie: %s" % movie.getFileName()) self._processMovie(movie) if self._doMovieFolderCleanUp(): self._cleanMovieFolder(movieFolder) # Mark this movie as finished open(movieDoneFn, 'w').close()
def processMovieStep(self, movieDict, hasAlignment): movie = Movie() movie.setAcquisition(Acquisition()) if hasAlignment: movie.setAlignment(MovieAlignment()) movie.setAttributesFromDict(movieDict, setBasic=True, ignoreMissing=True) movieFolder = self._getOutputMovieFolder(movie) movieFn = movie.getFileName() movieName = basename(movieFn) if (self.isContinued() and os.path.exists(self._getMovieDone(movie))): self.info("Skipping movie: %s, seems to be done" % movieFn) return # Clean old finished files pwutils.cleanPath(self._getMovieDone(movie)) if self._filterMovie(movie): pwutils.makePath(movieFolder) pwutils.createLink(movieFn, join(movieFolder, movieName)) if movieName.endswith('bz2'): newMovieName = movieName.replace('.bz2', '') # We assume that if compressed the name ends with .mrc.bz2 if not exists(newMovieName): self.runJob('bzip2', '-d -f %s' % movieName, cwd=movieFolder) elif movieName.endswith('tbz'): newMovieName = movieName.replace('.tbz', '.mrc') # We assume that if compressed the name ends with .tbz if not exists(newMovieName): self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder) elif movieName.endswith('.tif'): #FIXME: It seems that we have some flip problem with compressed # tif files, we need to check that newMovieName = movieName.replace('.tif', '.mrc') # we assume that if compressed the name ends with .tbz if not exists(newMovieName): self.runJob('tif2mrc', '%s %s' % (movieName, newMovieName), cwd=movieFolder) elif movieName.endswith('.txt'): # Support a list of frame as a simple .txt file containing # all the frames in a raw list, we could use a xmd as well, # but a plain text was choose to simply its generation movieTxt = os.path.join(movieFolder, movieName) with open(movieTxt) as f: movieOrigin = os.path.basename(os.readlink(movieFn)) newMovieName = movieName.replace('.txt', '.mrcs') ih = ImageHandler() for i, line in enumerate(f): if line.strip(): inputFrame = os.path.join(movieOrigin, line.strip()) ih.convert( inputFrame, (i + 1, os.path.join(movieFolder, newMovieName))) else: newMovieName = movieName if (self.CONVERT_TO_MRC and not (newMovieName.endswith("mrc") or newMovieName.endswith("mrcs"))): inputMovieFn = os.path.join(movieFolder, newMovieName) if inputMovieFn.endswith('.em'): inputMovieFn += ":ems" newMovieName = pwutils.replaceExt(newMovieName, self.CONVERT_TO_MRC) outputMovieFn = os.path.join(movieFolder, newMovieName) self.info("Converting movie '%s' -> '%s'" % (inputMovieFn, outputMovieFn)) ImageHandler().convertStack(inputMovieFn, outputMovieFn) # Just store the original name in case it is needed in _processMovie movie._originalFileName = pwobj.String(objDoStore=False) movie._originalFileName.set(movie.getFileName()) # Now set the new filename (either linked or converted) movie.setFileName(os.path.join(movieFolder, newMovieName)) self.info("Processing movie: %s" % movie.getFileName()) self._processMovie(movie) if pwutils.envVarOn('SCIPION_DEBUG_NOCLEAN'): self.info('Clean movie data DISABLED. ' 'Movie folder will remain in disk!!!') else: self.info("Erasing.....movieFolder: %s" % movieFolder) os.system('rm -rf %s' % movieFolder) # cleanPath(movieFolder) # Mark this movie as finished open(self._getMovieDone(movie), 'w').close()
def _getMicDone(self, mic): fn = mic.getFileName() extFn = getExt(fn) if extFn != ".mrc": fn = replaceExt(fn, "mrc") return self._getExtraPath('%s' % basename(fn))
def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='png', micIdSet=None): """Adds to self.thumbPaths the paths to the report thumbnails that come from the alignment and/or ctf protocol. ===== Params ===== - thumbsDone: how many thumbnails have already been generated. we will get paths starting from this index - ctfData: dict resulting from ctfMonitor.getData() - ext: extension of the thumbnail images. Defaults to png. - micIdSet: mic indexes to use """ # get psd thumbs from ctfData if ctfData is not None: for i in range(thumbsDone, len(ctfData[PSD_PATH])): psdPath = ctfData[PSD_PATH][i] movie = basename(os.path.dirname(psdPath)) psdThumb = join(PSD_THUMBS, "%s_%s" % (movie, pwutils.replaceExt(basename(psdPath), ext))) self.thumbPaths[PSD_THUMBS].append(psdThumb) self.thumbPaths[PSD_PATH].append(psdPath) # get alignment and mic thumbs if self.alignProtocol is not None: getMicFromCTF = False updatedProt = getUpdatedProtocol(self.alignProtocol) if hasattr(updatedProt, 'outputMicrographs'): outputSet = updatedProt.outputMicrographs if micIdSet is None: micIdSet = list(outputSet.getIdSet()) else: return elif self.ctfProtocol is not None: getMicFromCTF = True updatedProt = getUpdatedProtocol(self.ctfProtocol) if hasattr(updatedProt, 'outputCTF'): outputSet = updatedProt.outputCTF if micIdSet is None: micIdSet = list(outputSet.getIdSet()) else: return else: return for micId in micIdSet[thumbsDone:]: mic = outputSet[micId] if getMicFromCTF: mic = mic.getMicrograph() if hasattr(mic, 'thumbnail'): srcMicFn = abspath(mic.thumbnail.getFileName()) else: srcMicFn = abspath(mic.getFileName()) micThumbFn = join(MIC_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext)) self.thumbPaths[MIC_PATH].append(srcMicFn) self.thumbPaths[MIC_THUMBS].append(micThumbFn) shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None)) if shiftPlot is not None: shiftPath = "" if shiftPlot is None else abspath(shiftPlot.getFileName()) shiftCopy = "" if shiftPlot is None else join(SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext)) self.thumbPaths[SHIFT_PATH].append(shiftPath) self.thumbPaths[SHIFT_THUMBS].append(shiftCopy) else: if SHIFT_PATH in self.thumbPaths: self.thumbPaths.pop(SHIFT_PATH, None) if SHIFT_THUMBS in self.thumbPaths: self.thumbPaths.pop(SHIFT_THUMBS, None) self.thumbPaths[MIC_ID].append(micId) if self.ctfProtocol is None: def getMicPSDPath(mic): if hasattr(mic, 'psdJpeg'): return mic.psdJpeg.getFileName() elif hasattr(mic,'psdCorr'): return mic.psdCorr.getFileName() else: return None psdPath = getMicPSDPath(mic) psdThumb = None if psdPath ==None: psdThumb = join(PSD_THUMBS, pwutils.replaceExt(basename(psdPath), ext)) self.thumbPaths[PSD_THUMBS].append(psdThumb) self.thumbPaths[PSD_PATH].append(psdPath) else: if PSD_THUMBS in self.thumbPaths: self.thumbPaths.pop(PSD_THUMBS, None) if PSD_PATH in self.thumbPaths: self.thumbPaths.pop(PSD_PATH, None)
def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='png', micIdSet=None): """Adds to self.thumbPaths the paths to the report thumbnails that come from the alignment and/or ctf protocol. ===== Params ===== - thumbsDone: how many thumbnails have already been generated. we will get paths starting from this index - ctfData: dict resulting from ctfMonitor.getData() - ext: extension of the thumbnail images. Defaults to png. - micIdSet: mic indexes to use """ # get psd thumbs from ctfData if ctfData is not None: for i in range(thumbsDone, len(ctfData[PSD_PATH])): psdPath = ctfData[PSD_PATH][i] movie = basename(os.path.dirname(psdPath)) psdThumb = join( PSD_THUMBS, "%s_%s" % (movie, pwutils.replaceExt(basename(psdPath), ext))) self.thumbPaths[PSD_THUMBS].append(psdThumb) self.thumbPaths[PSD_PATH].append(psdPath) # get alignment and mic thumbs if self.alignProtocol is not None: getMicFromCTF = False updatedProt = getUpdatedProtocol(self.alignProtocol) if hasattr(updatedProt, 'outputMicrographs'): outputSet = updatedProt.outputMicrographs if micIdSet is None: micIdSet = list(outputSet.getIdSet()) else: return elif self.ctfProtocol is not None: getMicFromCTF = True updatedProt = getUpdatedProtocol(self.ctfProtocol) if hasattr(updatedProt, 'outputCTF'): outputSet = updatedProt.outputCTF if micIdSet is None: micIdSet = list(outputSet.getIdSet()) else: return else: return for micId in micIdSet[thumbsDone:]: mic = outputSet[micId] if getMicFromCTF: mic = mic.getMicrograph() if hasattr(mic, 'thumbnail'): srcMicFn = abspath(mic.thumbnail.getFileName()) else: srcMicFn = abspath(mic.getFileName()) micThumbFn = join(MIC_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext)) self.thumbPaths[MIC_PATH].append(srcMicFn) self.thumbPaths[MIC_THUMBS].append(micThumbFn) shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None)) if shiftPlot is not None: shiftPath = "" if shiftPlot is None else abspath( shiftPlot.getFileName()) shiftCopy = "" if shiftPlot is None else join( SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext)) self.thumbPaths[SHIFT_PATH].append(shiftPath) self.thumbPaths[SHIFT_THUMBS].append(shiftCopy) else: if SHIFT_PATH in self.thumbPaths: self.thumbPaths.pop(SHIFT_PATH, None) if SHIFT_THUMBS in self.thumbPaths: self.thumbPaths.pop(SHIFT_THUMBS, None) self.thumbPaths[MIC_ID].append(micId) if self.ctfProtocol is None: psdPath = mic.psdJpeg.getFileName() if hasattr( mic, 'psdJpeg') else mic.psdCorr.getFileName() psdThumb = join(PSD_THUMBS, pwutils.replaceExt(basename(psdPath), ext)) self.thumbPaths[PSD_THUMBS].append(psdThumb) self.thumbPaths[PSD_PATH].append(psdPath)