def show(self, form): prot = form.protocol micSet = prot.getInputMicrographs() if not micSet: print 'must specify input micrographs' return project = prot.getProject() micfn = micSet.getFileName() # Prepare a temporary folder to convert some input files # and put some of the intermediate result files coordsDir = project.getTmpPath(micSet.getName()) pwutils.cleanPath(coordsDir) pwutils.makePath(coordsDir) prot.convertInputs(coordsDir) pickerConfig = os.path.join(coordsDir, 'picker.conf') f = open(pickerConfig, "w") pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py') pickCmd = prot._getPickArgs(threshold=False, workingDir=coordsDir)[0] convertCmd = pw.join('apps', 'pw_convert.py') args = { "pickScript": pickScript, "pickCmd": pickCmd, "convertCmd": convertCmd, 'coordsDir': coordsDir, 'micsSqlite': micSet.getFileName(), 'thresholdLow': prot.thresholdLow, 'thresholdHigh': prot.thresholdHigh, "useGPU": prot.useGPU } f.write(""" parameters = thresholdLow,thresholdHigh thresholdLow.value = %(thresholdLow)s thresholdLow.label = Threshold Low thresholdLow.help = Low value cut-off thresholdHigh.value = %(thresholdHigh)s thresholdHigh.label = Threshold High thresholdHigh.help = High value cut-off autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh) convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s """ % args) f.close() process = CoordinatesObjectView(project, micfn, coordsDir, prot, mode=CoordinatesObjectView.MODE_AUTOMATIC, pickerProps=pickerConfig).show() process.wait() myprops = pwutils.readProperties(pickerConfig) if myprops['applyChanges'] == 'true': form.setVar('thresholdLow', myprops['thresholdLow.value']) form.setVar('thresholdHigh', myprops['thresholdHigh.value'])
def convertInputStep(self, micsId, refsId): """ This step will take of the convertions from the inputs. Micrographs: they will be linked if are in '.mrc' format, converted otherwise. References: will always be converted to '.mrc' format Mask: either converted ('.tif' format) or generated a circular one """ ih = em.ImageHandler() micDir = self._getTmpPath('micrographs') pwutils.makePath(micDir) for mic in self.getInputMicrographs(): # Create micrograph folder micName = mic.getFileName() # If micrographs are in .mrc format just link it # otherwise convert them outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(mic, outMic) refDir = self._getTmpPath('templates') pwutils.makePath(refDir) # We will always convert the templates, since # they can be in an stack and link will not be possible sometimes inputRefs = self.inputReferences.get() for i, ref in enumerate(inputRefs): outRef = join(refDir, 'ref%02d.mrc' % (i+1)) ih.convert(ref, outRef) self.createInputMasks(inputRefs)
def _estimateCtfList(self, micList, *args, **kwargs): """ Estimate several micrographs at once, probably a bit more efficient. """ try: micPath = self._getMicrographDir(micList[0]) if len(micList) > 1: micPath += ('-%04d' % micList[-1].getObjId()) pwutils.makePath(micPath) ih = emlib.image.ImageHandler() for mic in micList: micFn = mic.getFileName() # We convert the input micrograph on demand if not in .mrc downFactor = self.ctfDownFactor.get() micFnMrc = os.path.join(micPath, pwutils.replaceBaseExt(micFn, 'mrc')) if downFactor != 1: # Replace extension by 'mrc' cause there are some formats # that cannot be written (such as dm3) ih.scaleFourier(micFn, micFnMrc, downFactor) sps = self._params['scannedPixelSize'] * downFactor kwargs['scannedPixelSize'] = sps else: ih.convert(micFn, micFnMrc, emlib.DT_FLOAT) program, args = self._gctfProgram.getCommand(**kwargs) args += ' %s/*.mrc' % micPath self.runJob(program, args) # , env=gctf.Plugin.getEnviron()) def _getFile(micBase, suffix): return os.path.join(micPath, micBase + suffix) for mic in micList: micFn = mic.getFileName() micBase = pwutils.removeBaseExt(micFn) micFnMrc = _getFile(micBase, '.mrc') # Let's clean the temporary mrc micrograph pwutils.cleanPath(micFnMrc) # move output from tmp to extra micFnCtf = _getFile(micBase, self._gctfProgram.getExt()) micFnCtfLog = _getFile(micBase, '_gctf.log') micFnCtfFit = _getFile(micBase, '_EPA.log') micFnCtfOut = self._getPsdPath(micFn) micFnCtfLogOut = self._getCtfOutPath(micFn) micFnCtfFitOut = self._getCtfFitOutPath(micFn) pwutils.moveFile(micFnCtf, micFnCtfOut) pwutils.moveFile(micFnCtfLog, micFnCtfLogOut) pwutils.moveFile(micFnCtfFit, micFnCtfFitOut) pwutils.cleanPath(micPath) except: print("ERROR: Gctf has failed on %s/*.mrc" % micPath) import traceback traceback.print_exc()
def testWriteSetOfCoordinatesWithoutFlip(self): from collections import OrderedDict # Define a temporary sqlite file for micrographs fn = self.getOutputPath('convert_mics.sqlite') mics = emobj.SetOfMicrographs(filename=fn) # Create SetOfCoordinates data # Define a temporary sqlite file for coordinates fn = self.getOutputPath('convert_coordinates.sqlite') coordSet = emobj.SetOfCoordinates(filename=fn) coordSet.setBoxSize(60) coordSet.setMicrographs(mics) data = OrderedDict() data['006'] = [(30, 30)] data['016'] = [(40, 40)] micList = [] for key, coords in data.items(): mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key)) mics.append(mic) micList.append(mic) print("Adding mic: %s, id: %s" % (key, mic.getObjId())) for x, y in coords: coord = emobj.Coordinate(x=x, y=y) coord.setMicrograph(mic) coordSet.append(coord) # Get boxDirectory boxFolder = self.getOutputPath('boxFolder') os.mkdir(boxFolder) micFolder = self.getOutputPath('micFolder') pwutils.makePath(micFolder) # Invoke the write set of coordinates method convert.writeSetOfCoordinates(boxFolder, coordSet) convert.convertMicrographs(micList, micFolder) # Assert output of writesetofcoordinates for mic in micList: boxFile = os.path.join(boxFolder, convert.getMicIdName(mic, '.box')) self.assertTrue(os.path.exists(boxFile), 'Missing box file: %s' % boxFile) micFile = os.path.join(micFolder, convert.getMicIdName(mic, '.mrc')) self.assertTrue(os.path.exists(micFile), 'Missing box file: %s' % micFile) # Assert coordinates in box files fh = open(os.path.join(boxFolder, 'mic00001.box')) box1 = fh.readline() fh.close() box1 = box1.split('\t') self.assertEquals(box1[0], '0') self.assertEquals(box1[1], '964')
def pysegPostRec(self, outStar): # Generate output subtomo dir outDir = self._getExtraPath(POST_REC_OUT) makePath(outDir) # Script called Plugin.runPySeg(self, PYTHON, self._getCommand(outDir, outStar))
def testConvertCoords(self): boxSize = 100 boxDir = self.getOutputPath('boxDir') pwutils.makePath(boxDir) def _convert(coordsIn, yFlipHeight=None): tmpFile = os.path.join(boxDir, 'tmp.cbox') # Write input coordinates writer = convert.CoordBoxWriter(boxSize, yFlipHeight=yFlipHeight) writer.open(tmpFile) for x, y, _ in coordsIn: writer.writeCoord(emobj.Coordinate(x=x, y=y)) writer.close() reader = convert.CoordBoxReader(boxSize, yFlipHeight=yFlipHeight) reader.open(tmpFile) coordsOut = [c for c in reader.iterCoords()] reader.close() return coordsOut coordsIn = [(100, 100, 0.), (100, 200, 0.), (200, 100, 0.), (200, 200, 0.)] # Case 1: No flip coordsOut = _convert(coordsIn) for c1, c2 in zip(coordsIn, coordsOut): self.assertEqual(c1, c2) # Case 2: Flipping on Y coordsOut = _convert(coordsIn, yFlipHeight=300) for c1, c2 in zip(coordsIn, coordsOut): self.assertEqual(c1, c2)
def _insertInitialSteps(self): """ Override this function to insert some steps before the estimate ctfs steps. Should return a list of ids of the initial steps. """ pwutils.makePath(self._getExtraPath('DONE')) return []
def exportData(emxDir, inputSet, ctfSet=None, xmlFile='data.emx', binaryFile=None, doConvert=True): """ Export micrographs, coordinates or particles to EMX format. """ pwutils.cleanPath(emxDir) pwutils.makePath(emxDir) emxData = emxlib.EmxData() micSet=None if isinstance(inputSet, SetOfMicrographs): _micrographsToEmx(emxData, inputSet, emxDir, ctfSet, writeData=True) elif isinstance(inputSet, SetOfCoordinates): micSet = inputSet.getMicrographs() _micrographsToEmx(emxData, micSet, emxDir, ctfSet) _particlesToEmx(emxData, inputSet, micSet, writeImages=False) # _particlesToEmx(emxData, inputSet, None, micSet, doConvert=doConvert) elif isinstance(inputSet, SetOfParticles): if inputSet.hasCoordinates(): micSet = inputSet.getCoordinates().getMicrographs() _micrographsToEmx(emxData, micSet, emxDir, writeData=False) kwargs = {'writeImages': True} if binaryFile is None: kwargs['imagesPrefix'] = emxDir else: kwargs['imagesStack'] = join(emxDir, binaryFile) _particlesToEmx(emxData, inputSet, micSet, **kwargs) fnXml = join(emxDir, xmlFile) emxData.write(fnXml)
def _estimateCTF(self, micFn, micDir, micName): """ Run ctffind, 3 or 4, with required parameters """ # Create micrograph dir pwutils.makePath(micDir) downFactor = self.ctfDownFactor.get() micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc')) if downFactor != 1: #Replace extension by 'mrc' cause there are some formats that cannot be written (such as dm3) import pyworkflow.em.packages.xmipp3 as xmipp3 self.runJob("xmipp_transform_downsample", "-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor), env=xmipp3.getEnviron()) self._params['scannedPixelSize'] = self.inputMicrographs.get().getScannedPixelSize() * downFactor else: micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc")) em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT) # Update _params dictionary self._params['micFn'] = micFnMrc self._params['micDir'] = micDir self._params['ctffindOut'] = self._getCtfOutPath(micDir) self._params['ctffindPSD'] = self._getPsdPath(micDir) try: self.runJob(self._program, self._args % self._params) except Exception, ex: print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def msaStep(self): """ Run MSA on input particles. """ distances = ['EUCLIDIAN', 'CHISQUARE', 'MODULATION'] distance_name = distances[self.distanceType.get()] self._params.update({ 'msa_dir': self.MSA_DIR, 'msa_distance': distance_name, 'num_factors': self.numberOfFactors.get(), 'num_iter': self.numberOfIterations.get(), 'overcorrectionFactor': self.overcorrectionFactor.get(), 'mpi_procs': self.numberOfMpi.get() }) msaDir = self._getPath(self.MSA_DIR) if exists(msaDir): pwutils.cleanPath(msaDir) pwutils.makePath(msaDir) self.runTemplate('msa/msa-run.b', self._params)
def show(self, form): autopickProt = form.protocol micSet = autopickProt.getInputMicrographs() if not micSet: print 'must specify input micrographs' return project = autopickProt.getProject() micfn = micSet.getFileName() coordsDir = project.getTmpPath(micSet.getName()) cleanPath(coordsDir) makePath(coordsDir) # Get current values of the properties # micfn = os.path.join(coordsDir, 'micrographs.xmd') # writeSetOfMicrographs(micSet, micfn) dogpickerProps = os.path.join(coordsDir, 'picker.conf') f = open(dogpickerProps, "w") args = { "dogpicker": os.path.join(os.environ['DOGPICKER_HOME'], "ApDogPicker.py"), "convert": pw.join('apps', 'pw_convert.py'), 'coordsDir': coordsDir, 'micsSqlite': micSet.getFileName(), "diameter": autopickProt.diameter, "threshold": autopickProt.threshold, "apix": micSet.getSamplingRate() } f.write(""" parameters = diameter,threshold diameter.value = %(diameter)s diameter.label = Diameter diameter.help = some help threshold.value = %(threshold)s threshold.label = Threshold threshold.help = some help autopickCommand = %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameter) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s """ % args) f.close() process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt, pickerProps=dogpickerProps).show() process.wait() # Check if the wizard changes were accepted or just canceled myprops = readProperties(dogpickerProps) if myprops['applyChanges'] == 'true': form.setVar('diameter', myprops['diameter.value']) form.setVar('threshold', myprops['threshold.value'])
def runExecuteCtfGroupsStep(self, **kwargs): makePath(self.ctfGroupDirectory) self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory) # printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line if not self.doCTFCorrection: md = xmipp.MetaData(self.selFileName) block_name = self._getBlockFileName(ctfBlockName, 1, self._getFileName('imageCTFpairs')) md.write(block_name) self._log.info("Written a single CTF group to file: '%s'" % block_name) self.numberOfCtfGroups.set(1) else: self._log.info( '*********************************************************************' ) self._log.info('* Make CTF groups') # remove all entries not present in sel file by # join between selfile and metadatafile mdCtfData = xmipp.MetaData() mdCtfData.read(self.ctfDatName) mdSel = xmipp.MetaData() mdSel.read(self.selFileName) mdCtfData.intersection(mdSel, xmipp.MDL_IMAGE) tmpCtfDat = self.ctfDatName mdCtfData.write(tmpCtfDat) args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s' args += ' --sampling_rate %(sampling)s' params = { 'tmpCtfDat': tmpCtfDat, 'ctffile': self._getFileName('ctfGroupBase') + ':stk', 'wiener': self.wienerConstant.get(), 'pad': self.paddingFactor.get(), 'sampling': self.resolSam } if self.inputParticles.get().isPhaseFlipped(): args += ' --phase_flipped ' if self.doAutoCTFGroup: args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s' params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get() params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get() else: if exists(self.setOfDefocus.get()): args += ' --split %(setOfDefocus)s' params['setOfDefocus'] = self.setOfDefocus.get() self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs) auxMD = xmipp.MetaData("numberGroups@" + self._getFileName('cTFGroupSummary')) self.numberOfCtfGroups.set( auxMD.getValue(xmipp.MDL_COUNT, auxMD.firstObject())) self._store(self.numberOfCtfGroups)
def writeCtfStarStep(self): inputCTF = self.inputCTF.get() if self.micrographSource == 0: # same as CTF estimation ctfMicSet = inputCTF.getMicrographs() else: ctfMicSet = self.inputMicrographs.get() micSet = SetOfMicrographs(filename=':memory:') psd = inputCTF.getFirstItem().getPsdFile() hasPsd = psd and os.path.exists(psd) if hasPsd: psdPath = self._getPath('PSD') pwutils.makePath(psdPath) print "Writing PSD files to %s" % psdPath for ctf in inputCTF: # Get the corresponding micrograph mic = ctfMicSet[ctf.getObjId()] if mic is None: print("Skipping CTF id: %s, it is missing from input " "micrographs. " % ctf.getObjId()) continue micFn = mic.getFileName() if not os.path.exists(micFn): print "Skipping micrograph %s, it does not exists. " % micFn continue mic2 = mic.clone() mic2.setCTF(ctf) if hasPsd: psdFile = ctf.getPsdFile() newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName()) if not os.path.exists(psdFile): print "PSD file %s does not exits" % psdFile print "Skipping micrograph %s" % micFn continue pwutils.copyFile(psdFile, newPsdFile) ctf.setPsdFile(newPsdFile) micSet.append(mic2) starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId()) print "Writing set: %s" % inputCTF print " to: %s" % starFile acq = ctfMicSet.getAcquisition() self.samplingRate = ctfMicSet.getSamplingRate() mag = acq.getMagnification() self.detectorPixelSize = 1e-4 * self.samplingRate * mag writeSetOfMicrographs(micSet, starFile, preprocessImageRow=self.preprocessMicrograph) # Let's create a link from the project roots to facilitate the import # of the star file into a Relion project pwutils.createLink(starFile, os.path.basename(starFile))
def show(self, form): prot = form.protocol micSet = prot.getInputMicrographs() if not micSet: print 'must specify input micrographs' return project = prot.getProject() micfn = micSet.getFileName() # Prepare a temporary folder to convert some input files # and put some of the intermediate result files coordsDir = project.getTmpPath(micSet.getName()) pwutils.cleanPath(coordsDir) pwutils.makePath(coordsDir) prot.convertInputs(coordsDir) pickerConfig = os.path.join(coordsDir, 'picker.conf') f = open(pickerConfig, "w") pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py') pickCmd = prot.getArgs(threshold=False, workingDir=coordsDir) convertCmd = pw.join('apps', 'pw_convert.py') args = { "pickScript": pickScript, "pickCmd": pickCmd, "convertCmd": convertCmd, 'coordsDir': coordsDir, 'micsSqlite': micSet.getFileName(), 'thresholdLow': prot.thresholdLow, 'thresholdHigh': prot.thresholdHigh, "useGPU": prot.useGPU } f.write(""" parameters = thresholdLow,thresholdHigh thresholdLow.value = %(thresholdLow)s thresholdLow.label = Threshold Low thresholdLow.help = Low value cut-off thresholdHigh.value = %(thresholdHigh)s thresholdHigh.label = Threshold High thresholdHigh.help = High value cut-off autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh) convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s """ % args) f.close() process = CoordinatesObjectView(project, micfn, coordsDir, prot, pickerProps=pickerConfig).show() process.wait() myprops = pwutils.readProperties(pickerConfig) if myprops['applyChanges'] == 'true': form.setVar('thresholdLow', myprops['thresholdLow.value']) form.setVar('thresholdHigh', myprops['thresholdHigh.value'])
def _writeXmippCoords(self, coordSet): micSet = self.getInputMicrographs() coordPath = self._getTmpPath('xmipp_coordinates') pwutils.cleanPath(coordPath) pwutils.makePath(coordPath) micPath = micSet.getFileName() convert.writeSetOfCoordinatesXmipp(coordPath, coordSet, ismanual=False) return micPath, coordPath
def _convertVolumes(self): ih = ImageHandler() makePath(self._getExtraPath('input')) inputVols = self.inputVolumes.get() for vol in inputVols: map = ih.read(vol) outputFn = self._getOutputFn(vol.getObjId()) map.write(outputFn)
def _createFolder(self, p): if os.path.exists(p): raise Exception("Path '%s' already exists.\n" % p) # Create the project path sys.stdout.write("Creating path '%s' ... " % p) pwutils.makePath(p) os.chmod(p, 0o0775) sys.stdout.write("DONE\n")
def initializeParams(self): self.finished = False self.insertedDict = {} self.initializeRejDict() self.setSecondaryAttributes() self.ctfFn1 = self.inputCTF.get().getFileName() self.allCtf1 = {} pwutils.makePath(self._getExtraPath('DONE'))
def _processMovie(self, movie): movieFolder = self._getOutputMovieFolder(movie) inputStar = os.path.join(movieFolder, '%s_input.star' % self._getMovieRoot(movie)) pwutils.makePath(os.path.join(movieFolder, 'output')) og = OpticsGroups.fromImages(self.inputMovies.get()) writer = convert.createWriter(optics=og) # Let's use only the basename, since we will launch the command # from the movieFolder movie.setFileName(os.path.basename(movie.getFileName())) writer.writeSetOfMovies([movie], inputStar) # The program will run in the movie folder, so let's put # the input files relative to that args = "--i %s --o output/ " % os.path.basename(inputStar) args += "--use_own " f0, fN = self._getRange(movie) args += "--first_frame_sum %d --last_frame_sum %d " % (f0, fN) args += "--bin_factor %f --bfactor %d " % (self.binFactor, self.bfactor) args += "--angpix %0.5f " % (movie.getSamplingRate()) args += "--patch_x %d --patch_y %d " % (self.patchX, self.patchY) args += "--group_frames %d " % self.groupFrames args += "--j %d " % self.numberOfThreads inputMovies = self.inputMovies.get() if inputMovies.getGain(): args += ' --gainref "%s" ' % inputMovies.getGain() args += ' --gain_rot %d ' % self.gainRot args += ' --gain_flip %d ' % self.gainFlip if self.IS_GT30(): if self.defectFile.get(): args += ' --defect_file "%s" ' % self.defectFile.get() if self._savePsSum(): args += ' --grouping_for_ps %d ' % self._calcPsDose() if self.doDW: args += "--dose_weighting " preExp, dose = self._getCorrectedDose(self.inputMovies.get()) args += "--dose_per_frame %f " % dose args += "--preexposure %f " % preExp if self.saveNonDW: args += " --save_noDW " if self.extraParams.hasValue(): args += " " + self.extraParams.get() try: self.runJob(self._getProgram(), args, cwd=movieFolder) self._computeExtra(movie) self._moveFiles(movie) except: print("ERROR: processing movie: ", movie.getFileName())
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile( self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile( self._getPath( 'movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation( mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation( mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def runCase(self, args, mpi=0, changeDir=False, preruns=None, postruns=None, validate=None, outputs=None, random=False): # Retrieve the correct case number from the test name id # We asumme here that 'test_caseXXX' should be in the name caseId = unittest.TestCase.id(self) if not 'test_case' in caseId: raise Exception("'test_case' string should be in the test function name followed by a number") _counter = int(caseId.split('test_case')[1]) self._testDir = self.dataset.getPath() self.outputDir = os.path.join('tmpLink', '%s_%02d' % (self.program, _counter)) self.outputDirAbs = os.path.join(self._testDir, self.outputDir) self.goldDir = os.path.join(self._testDir, 'gold', '%s_%02d' % (self.program, _counter)) # Clean and create the program output folder if not exists pwutils.cleanPath(self.outputDirAbs) pwutils.makePath(self.outputDirAbs) # Change to tests root folder (self._testDir) cwd = os.getcwd() os.chdir(self._testDir) if preruns: self._runCommands(preruns, 'preruns') if mpi: cmd = "mpirun -np %d `which %s`" % (mpi, self.program) else: cmd = self.program args = self._parseArgs(args) if changeDir: cmd = "cd %s ; %s %s > stdout.txt 2> stderr.txt" % (self.outputDir, cmd, args) else: cmd = "%s %s > %s/stdout.txt 2> %s/stderr.txt" % (cmd, args, self.outputDir, self.outputDir) print " Command: " print " ", pwutils.green(cmd) #run the test itself command = Command(cmd, env=self.env) self._command = command try: command.run(timeout=self._timeout) except KeyboardInterrupt: command.terminate() if postruns: self._runCommands(postruns, 'postruns') if outputs: self._checkOutputs(outputs,random) if validate: validate() os.chdir(cwd)
def _writeXmippCoords(self, coordSet): micSet = self.getInputMicrographs() coordPath = self._getTmpPath('xmipp_coordinates') pwutils.cleanPath(coordPath) pwutils.makePath(coordPath) import pyworkflow.em.packages.xmipp3 as xmipp3 micPath = micSet.getFileName() xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False) return micPath, coordPath
def exportCoordsStep(self, coordsId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. """ pwutils.cleanPath(self._getExportPath()) pwutils.makePath(self._getExportPath()) convert.writeSetOfCoordinates(self._getExportPath(), self.getCoords(), self._getMicPos)
def writeCtfStarStep(self): inputCTF = self.inputCTF.get() if self.micrographSource == 0: # same as CTF estimation ctfMicSet = inputCTF.getMicrographs() else: ctfMicSet = self.inputMicrographs.get() micSet = SetOfMicrographs(filename=':memory:') psd = inputCTF.getFirstItem().getPsdFile() hasPsd = psd and os.path.exists(psd) if hasPsd: psdPath = self._getPath('PSD') pwutils.makePath(psdPath) print "Writing PSD files to %s" % psdPath for ctf in inputCTF: # Get the corresponding micrograph mic = ctfMicSet[ctf.getObjId()] if mic is None: print( "Skipping CTF id: %s, it is missing from input " "micrographs. " % ctf.getObjId()) continue micFn = mic.getFileName() if not os.path.exists(micFn): print "Skipping micrograph %s, it does not exists. " % micFn continue mic2 = mic.clone() mic2.setCTF(ctf) if hasPsd: psdFile = ctf.getPsdFile() newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName()) if not os.path.exists(psdFile): print "PSD file %s does not exits" % psdFile print "Skipping micrograph %s" % micFn continue pwutils.copyFile(psdFile, newPsdFile) ctf.setPsdFile(newPsdFile) micSet.append(mic2) starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId()) print "Writing set: %s" % inputCTF print " to: %s" % starFile writeSetOfMicrographs(micSet, starFile, preprocessImageRow=self.preprocessMicrograph) # Let's create a link from the project roots to facilitate the import # of the star file into a Relion project pwutils.createLink(starFile, os.path.basename(starFile))
def pysegPicking(self): # Generate output dir outDir = self._getExtraPath() makePath(outDir) # Generate slices xml self._createPickingXmlFile(Plugin.getHome(PICKING_SLICES), outDir) # Script called Plugin.runPySeg(self, PYTHON, self._getPickingCommand(outDir))
def _writeXmippCoords(self, coordSet): micSet = self.getInputMicrographs() coordPath = self._getTmpPath('xmipp_coordinates') pwutils.cleanPath(coordPath) pwutils.makePath(coordPath) import pyworkflow.em.packages.xmipp3 as xmipp3 micPath = os.path.join(coordPath, 'micrographs.xmd') xmipp3.writeSetOfMicrographs(micSet, micPath) xmipp3.writeSetOfCoordinates(coordPath, coordSet) return micPath, coordPath
def convertInputStep(self, protId): pwutils.makePath(self._getInputPath()) protRef = self.protRefine.get() vols = protRef.getFinalVolumes() # final, half1, half2 ih = ImageHandler() vols.append(self.solventMask.get()) for vol, key in zip(vols, ['finalVolume', 'half1', 'half2', 'mask']): ih.convert(vol, self._getFileName(key))
def _moveCoordsToInfo(self, tomo): fnCoor = '*%s_info.json' % pwutils.removeBaseExt(tomo.getFileName().split("__")[0]) pattern = os.path.join(self.path, fnCoor) files = glob.glob(pattern) if files: infoDir = pwutils.join(os.path.abspath(self.path), 'info') pathCoor = os.path.join(infoDir, os.path.basename(files[0])) pwutils.makePath(infoDir) copyFile(files[0], pathCoor)
def show(self, form): autopickProt = form.protocol micSet = autopickProt.getInputMicrographs() if not micSet: print 'must specify input micrographs' return project = autopickProt.getProject() micfn = micSet.getFileName() coordsDir = project.getTmpPath(micSet.getName()) cleanPath(coordsDir) makePath(coordsDir) pickerProps = os.path.join(coordsDir, 'picker.conf') f = open(pickerProps, "w") params = ['boxSize', 'lowerThreshold', 'higherThreshold', 'gaussWidth'] args = { "params": ','.join(params), "preprocess" : os.path.join(os.environ['SCIPION_HOME'], "scipion sxprocess.py"), "picker" : os.path.join(os.environ['SCIPION_HOME'], "scipion e2boxer.py"), "convert" : os.path.join(os.environ['SCIPION_HOME'], os.path.join('pyworkflow','apps', 'pw_convert.py')), 'coordsDir':coordsDir, 'micsSqlite': micSet.getFileName(), "boxSize": autopickProt.boxSize, "lowerThreshold": autopickProt.lowerThreshold, "higherThreshold": autopickProt.higherThreshold, "gaussWidth": autopickProt.gaussWidth, "extraParams":autopickProt.extraParams } f.write(""" parameters = %(params)s boxSize.value = %(boxSize)s boxSize.label = Box Size boxSize.help = some help lowerThreshold.value = %(lowerThreshold)s lowerThreshold.label = Lower Threshold lowerThreshold.help = some help higherThreshold.help = some help higherThreshold.value = %(higherThreshold)s higherThreshold.label = Higher Threshold gaussWidth.help = some help gaussWidth.value = %(gaussWidth)s gaussWidth.label = Gauss Width runDir = %(coordsDir)s preprocessCommand = %(preprocess)s demoparms --makedb=thr_low=%%(lowerThreshold):thr_hi=%%(higherThreshold):boxsize=%%(boxSize):gauss_width=%%(gaussWidth):%(extraParams)s autopickCommand = %(picker)s --gauss_autoboxer=demoparms --write_dbbox --boxsize=%%(boxSize) --norm=normalize.ramp.normvar %%(micrograph) convertCommand = %(convert)s --coordinates --from eman2 --to xmipp --input %(micsSqlite)s --output %(coordsDir)s """ % args) f.close() process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt, pickerProps=pickerProps).show() process.wait() myprops = readProperties(pickerProps) for param in params: form.setVar(param, myprops[param + '.value'])
def convertImagesStep(self): partSet = self._getInputParticles() partAlign = partSet.getAlignment() storePath = self._getExtraPath("particles") makePath(storePath) writeSetOfParticles(partSet, storePath, alignType=partAlign) if self.useInputBispec and self.inputBispec is not None: print( "Skipping CTF estimation since input bispectra were provided") self.skipctf.set(True) if not self.skipctf: program = eman2.Plugin.getProgram('e2ctf.py') acq = partSet.getAcquisition() args = " --voltage %d" % acq.getVoltage() args += " --cs %f" % acq.getSphericalAberration() args += " --ac %f" % (100 * acq.getAmplitudeContrast()) args += " --threads=%d" % self.numberOfThreads.get() if not partSet.isPhaseFlipped(): args += " --phaseflip" args += " --computesf --apix %f" % partSet.getSamplingRate() args += " --allparticles --autofit --curdefocusfix --storeparm -v 8" self.runJob(program, args, cwd=self._getExtraPath(), numberOfMpi=1, numberOfThreads=1) program = eman2.Plugin.getProgram('e2buildsets.py') args = " --setname=inputSet --allparticles --minhisnr=-1" self.runJob(program, args, cwd=self._getExtraPath(), numberOfMpi=1, numberOfThreads=1) if self.useInputBispec: prot = self.inputBispec.get() prot._createFilenameTemplates() bispec = prot.outputParticles_flip_bispec if not bispec.getSize() == partSet.getSize(): raise Exception( 'Input particles and bispectra sets have different size!') # link bispec hdf files and lst file pattern = prot._getExtraPath('particles/*__ctf_flip_bispec.hdf') print("\nLinking bispectra input files...") for fn in sorted(glob(pattern)): newFn = join(self._getExtraPath('particles'), basename(fn)) createLink(fn, newFn) print(" %s -> %s" % (fn, newFn)) lstFn = prot._getFileName('partSetFlipBispec') newLstFn = self._getFileName('partBispecSet') createLink(lstFn, newLstFn)
def _insertAllSteps(self): pwutils.makePath(self._getExtraPath('inputCoords')) pwutils.makePath(self._getExtraPath('outputCoords')) coordinates = self.inputCoordinates.get() self.tomos = coordinates.getPrecedents() self._insertFunctionStep('computeParams', coordinates) if self.outliers.get(): self._insertFunctionStep('detectOutliers') if self.carbon.get(): self._insertFunctionStep('detectCarbonCloseness', coordinates) self._insertFunctionStep('createOutputStep', coordinates)
def show(self, form): autopickProt = form.protocol micSet = autopickProt.getInputMicrographs() if not micSet: print('must specify input micrographs') return project = autopickProt.getProject() micfn = micSet.getFileName() coordsDir = project.getTmpPath(micSet.getName()) cleanPath(coordsDir) makePath(coordsDir) # Get current values of the properties # micfn = os.path.join(coordsDir, 'micrographs.xmd') # writeSetOfMicrographs(micSet, micfn) dogpickerProps = os.path.join(coordsDir, 'picker.conf') f = open(dogpickerProps, "w") args = { "dogpicker": os.path.join(Plugin.getHome(), "ApDogPicker.py"), "convert": 'emconvert', 'coordsDir': coordsDir, 'micsSqlite': micSet.getFileName(), "diameter": autopickProt.diameter, "threshold": autopickProt.threshold, "apix": micSet.getSamplingRate() } f.write(""" parameters = diameterA,threshold diameterA.value = %(diameter)s diameterA.label = Diameter in A diameterA.help = Sampling rate is %(apix)s threshold.value = %(threshold)s threshold.label = Threshold threshold.help = Threshold in standard deviations above the mean autopickCommand = python2 %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameterA) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s """ % args) f.close() process = CoordinatesObjectView( project, micfn, coordsDir, autopickProt, mode=CoordinatesObjectView.MODE_AUTOMATIC, pickerProps=dogpickerProps).show() process.wait() # Check if the wizard changes were accepted or just canceled myprops = readProperties(dogpickerProps) if myprops['applyChanges'] == 'true': form.setVar('diameter', myprops['diameterA.value']) form.setVar('threshold', myprops['threshold.value'])
def convertInputStep(self): """ This step will take of the conversions from the inputs. Micrographs: they will be linked if are in '.mrc' format, converted otherwise. References: will always be converted to '.mrcs' format """ micDir = self.getMicrographsDir() # put output and mics in extra dir pwutils.makePath(micDir) # We will always convert the templates to mrcs stack self.convertReferences(self._getExtraPath('references.mrcs')) # Convert input coords for exclusive picking self.convertCoordinates(micDir)
def _convertInputStep(self): pwutils.makePath(self._getExtraPath('DONE')) movs = self.inputMovies.get() # Convert gain gain = movs.getGain() movs.setGain(self.__convertCorrectionImage(gain)) # Convert dark dark = movs.getDark() movs.setDark(self.__convertCorrectionImage(dark))
def convertInputStep(self): """ This step will take of the conversions from the inputs. Micrographs: they will be linked if are in '.mrc' format, converted otherwise. References: will always be converted to '.mrcs' format """ pwutils.makePath(self.getMicrographsDir()) # put output and mics in extra dir # We will always convert the templates to mrcs stack self.convertReferences(self._getReferencesFn()) # Write defects star file if necessary if self.exclusive and self.inputDefects.get(): writeDefectsFile(self.inputDefects.get(), self._getDefectsFn())
def __init__(self, srcPath, dstPath, copyFiles=False): self._src = srcPath self._dst = dstPath self._processed = set() if copyFiles: pwutils.makePath(dstPath) self.transform = self._copyFile else: pwutils.makePath(os.path.dirname(srcPath)) pwutils.createLink(srcPath, dstPath) self.transform = self._addPrefix
def convertInputStep(self, micsId, refsId): pwutils.makePath( self._getExtraPath('DONE')) # Required to report finished inputRefs = self.getInputReferences() if self.useInputReferences(): relion.convert.writeReferences(inputRefs, self._getPath('reference_2d'), useBasename=True) else: ImageHandler().convert(inputRefs, self._getPath('reference_3d.mrc'))
def runExecuteCtfGroupsStep(self, **kwargs): makePath(self.ctfGroupDirectory) self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory) # printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line if not self.doCTFCorrection: md = xmipp.MetaData(self.selFileName) block_name = self._getBlockFileName(ctfBlockName, 1, self._getFileName('imageCTFpairs')) md.write(block_name) self._log.info("Written a single CTF group to file: '%s'" % block_name) self.numberOfCtfGroups.set(1) else: self._log.info('*********************************************************************') self._log.info('* Make CTF groups') # remove all entries not present in sel file by # join between selfile and metadatafile mdCtfData = xmipp.MetaData() mdCtfData.read(self.ctfDatName) mdSel = xmipp.MetaData(); mdSel.read(self.selFileName) mdCtfData.intersection(mdSel, xmipp.MDL_IMAGE) tmpCtfDat = self.ctfDatName mdCtfData.write(tmpCtfDat) args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s' args += ' --sampling_rate %(sampling)s' params = {'tmpCtfDat' : tmpCtfDat, 'ctffile' : self._getFileName('ctfGroupBase') + ':stk', 'wiener' : self.wienerConstant.get(), 'pad' : self.paddingFactor.get(), 'sampling' : self.resolSam } if self.inputParticles.get().isPhaseFlipped(): args += ' --phase_flipped ' if self.doAutoCTFGroup: args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s' params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get() params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get() else: if exists(self.setOfDefocus.get()): args += ' --split %(setOfDefocus)s' params['setOfDefocus'] = self.setOfDefocus.get() self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs) auxMD = xmipp.MetaData("numberGroups@" + self._getFileName('cTFGroupSummary')) self.numberOfCtfGroups.set(auxMD.getValue(xmipp.MDL_COUNT, auxMD.firstObject())) self._store(self.numberOfCtfGroups)
def convertInputStep(self): printLicense() self.info('Wrinting pytom xml file: ' + self.volXml) volsDir = self._getExtraPath('inputVolumes') pwutils.makePath(volsDir) writeSetOfVolumes(self.inputVolumes.get(), self.volXml, volsDir) if self.provideReferences: ih = em.ImageHandler() self.info('Converting input references') for r, vol in enumerate(self.inputReferences.get()): refFn = self._getExtraPath('reference_%02d.mrc' % (r+1)) ih.convert(vol, refFn)
def writeSetOfCoordinates3D(self): info_path = self._getExtraPath('info') pwutils.makePath(info_path) coords = self.inputCoordinates.get() tomos = coords.getPrecedents() tltSeries = recoverTSFromObj(coords, self) self.json_files, self.tomo_files = jsonFilesFromSet(tomos, info_path) _ = setCoords3D2Jsons(self.json_files, coords) _ = tltParams2Json(self.json_files, tltSeries, mode="a") if self.inputCTF.get() is not None: _ = ctf2Json(self.json_files, self.inputCTF.get(), mode='a')
def _insertAllSteps(self): numTomo = 0 makePath(self._getPredictConfDir()) # Insert processing steps for evenTomo, oddTomo in zip(self.even.get(), self.odd.get()): self._insertFunctionStep(self.preparePredictStep, evenTomo.getFileName(), oddTomo.getFileName(), numTomo) self._insertFunctionStep(self.predictStep, numTomo) numTomo += 1 self._insertFunctionStep(self.createOutputStep)
def convertInputStep(self): printLicense() self.info('Writing pytom xml file: ' + self.volXml) volsDir = self._getExtraPath('inputVolumes') pwutils.makePath(volsDir) writeSetOfVolumes(self.inputVolumes.get(), self.volXml, volsDir) ih = em.ImageHandler() self.info('Converting input reference to: ' + self.refVol) ih.convert(self.inputReference.get(), self.refVol) self.info('Converting input mask to: ' + self.maskVol) ih.convert(self.alignmentMask.get(), self.maskVol)
def setUp(self): """Setup actions for the html report: create directories to store thumbnails, check if thumbnails are already generated in the alignment protocol. """ # make report folders pwutils.makePath(join(self.reportDir, MIC_THUMBS), join(self.reportDir, PSD_THUMBS), join(self.reportDir, SHIFT_THUMBS)) # check if align protocol already has thumbnails if (hasattr(self.alignProtocol, 'doComputeMicThumbnail') and self.alignProtocol._doComputeMicThumbnail()): self.micThumbSymlinks = True
def show(self, form): autopickProt = form.protocol micSet = autopickProt.getInputMicrographs() if not micSet: print 'must specify input micrographs' return project = autopickProt.getProject() micfn = micSet.getFileName() coordsDir = project.getTmpPath(micSet.getName()) cleanPath(coordsDir) makePath(coordsDir) # Get current values of the properties # micfn = os.path.join(coordsDir, 'micrographs.xmd') # writeSetOfMicrographs(micSet, micfn) dogpickerProps = os.path.join(coordsDir, 'picker.conf') f = open(dogpickerProps, "w") args = { "dogpicker" : os.path.join(os.environ['DOGPICKER_HOME'], "ApDogPicker.py"), "convert" : pw.join('apps', 'pw_convert.py'), 'coordsDir': coordsDir, 'micsSqlite': micSet.getFileName(), "diameter": autopickProt.diameter, "threshold": autopickProt.threshold, "apix": micSet.getSamplingRate() } f.write(""" parameters = diameter,threshold diameter.value = %(diameter)s diameter.label = Diameter diameter.help = some help threshold.value = %(threshold)s threshold.label = Threshold threshold.help = some help autopickCommand = %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameter) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s """ % args) f.close() process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt, mode=CoordinatesObjectView.MODE_AUTOMATIC, pickerProps=dogpickerProps).show() process.wait() # Check if the wizard changes were accepted or just canceled myprops = readProperties(dogpickerProps) if myprops['applyChanges'] == 'true': form.setVar('diameter', myprops['diameter.value']) form.setVar('threshold', myprops['threshold.value'])
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def testImportFromCTffind4Conflict(self): micsPath = os.path.abspath(self.proj.getTmpPath('micrographs')) ctfsPath = os.path.abspath(self.proj.getTmpPath('ctfs')) pwutils.makePath(micsPath) pwutils.makePath(ctfsPath) micDict = {'BPV_1386': 'mic1', 'BPV_1387': 'mic10', 'BPV_1388': 'mic2'} pattern = 'micrographs/%s.mrc' for k, v in micDict.iteritems(): # Create a micrograph link with a given naming convention pwutils.createAbsLink(self.dsXmipp.getFile(pattern % k), self.proj.getTmpPath(pattern % v)) pwutils.createAbsLink(self.dsGrigorieff.getFile('ctffind4/%s' % k), self.proj.getTmpPath('ctfs/%s' % v)) protCTF = self.newProtocol(ProtImportCTF, importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF, filesPath=ctfsPath, filesPattern='mic*/*txt') protCTF.inputMicrographs.set(self.protImport.outputMicrographs) protCTF.setObjLabel('import from ctffind3 - empty') self.launchProtocol(protCTF) # FIXME: After the execution of the above case, some empty sets # are created, if there is no matching, there should not be any output # Let's import now the correct names, but with the problematic matching protImport2 = self.newProtocol(ProtImportMicrographs, filesPath=micsPath, filesPattern='mic*mrc', samplingRate=1.237, voltage=300) self.launchProtocol(protImport2) protCTF2 = self.newProtocol(ProtImportCTF, importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF, filesPath=ctfsPath, filesPattern='mic*/*txt') protCTF2.inputMicrographs.set(protImport2.outputMicrographs) protCTF2.setObjLabel('import from ctffind3 - match') self.launchProtocol(protCTF2) # The whole set (3 items) should find its corresponding CTF self.assertEqual(protCTF2.outputCTF.getSize(), 3)
def _writeRefinementScripts(self): """ Write the needed scripts to run refinement and substitute some values. """ refPath = self._getExtraPath('Refinement') pwutils.makePath(refPath) def path(p): """ Escape path with '' and add ../ """ return "'%s'" % join('..', p) def script(name, paramsDict={}): outputScript=join(refPath, name) writeScript(getScript('projmatch', 'Refinement', name), outputScript, paramsDict) nIter = self.numberOfIterations.get() def getListStr(valueStr): return "'%s'" % ','.join(pwutils.getListFromValues(valueStr, nIter)) diam = int(self.radius.get() * 2 * self.inputParticles.get().getSamplingRate()) params = {'[shrange]': self.alignmentShift.get(), '[iter-end]': self.numberOfIterations.get(), '[diam]': diam, '[win-frac]': self.winFrac.get(), '[converg]': self.convergence.get(), '[small-ang]': '1' if self.smallAngle else '0', '[ang-steps]': getListStr(self.angSteps.get()), '[ang-limits]': getListStr(self.angLimits.get()), '[ang-step-sm]': '(%0.2f)' % self.angStepSm.get(), '[theta-range]': '(%0.2f)' % self.thetaRange.get(), '[vol_orig]': path('vol001'), '[sel_group_orig]': path('sel_group'), '[sel_particles_orig]': path('group{***[grp]}_selfile'), '[group_align_orig]': path('group{***[grp]}_align'), '[unaligned_images_orig]': path('group{***[grp]}_stack'), } script('refine_settings.pam', params) for s in ['refine', 'prepare', 'grploop', 'mergegroups', 'enhance', 'endmerge', 'smangloop', 'endrefine']: script('%s.pam' % s)
def msaStep(self): """ Run MSA on input particles. """ distances = ['EUCLIDIAN', 'CHISQUARE', 'MODULATION'] distance_name = distances[self.distanceType.get()] self._params.update({'msa_dir': self.MSA_DIR, 'msa_distance': distance_name, 'num_factors': self.numberOfFactors.get(), 'num_iter': self.numberOfIterations.get(), 'overcorrectionFactor': self.overcorrectionFactor.get(), 'mpi_procs': self.numberOfMpi.get() }) msaDir = self._getPath(self.MSA_DIR) if exists(msaDir): pwutils.cleanPath(msaDir) pwutils.makePath(msaDir) self.runTemplate('msa/msa-run.b', self._params)
def createInputMasks(self, inputRefs): """ Create the needed mask for picking. We should either generate a circular mask, or convert the inputs (one or just one per reference 2d) """ maskSchDir = self._getTmpPath('maskSch') pwutils.makePath(maskSchDir) ih = em.ImageHandler() if self.maskType == MASK_CIRCULAR: if self.maskRadius < 0: # usually -1 radius = inputRefs.getDim()[0]/2 # use half of input dim else: radius = self.maskRadius.get() outMask = join(maskSchDir, 'ref01.tif') ih.createCircularMask(radius, inputRefs.getFirstItem(), outMask) else: for i, mask in enumerate(self.inputMasks): outMask = join(maskSchDir, 'ref%02d.tif' % (i+1)) ih.convert(mask.get(), outMask)
def _estimateCTF(self, micFn, micDir, micName): """ Run ctffind, 3 or 4, with required parameters """ doneFile = os.path.join(micDir, 'done.txt') if self.isContinued() and os.path.exists(doneFile): return try: # Create micrograph dir pwutils.makePath(micDir) downFactor = self.ctfDownFactor.get() scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize() micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc')) if downFactor != 1: # Replace extension by 'mrc' because there are some formats # that cannot be written (such as dm3) import pyworkflow.em.packages.xmipp3 as xmipp3 args = "-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor) self.runJob("xmipp_transform_downsample", args, env=xmipp3.getEnviron()) self._params['scannedPixelSize'] = scannedPixelSize * downFactor else: ih = em.ImageHandler() if ih.existsLocation(micFn): micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc")) ih.convert(micFn, micFnMrc, em.DT_FLOAT) else: print >> sys.stderr, "Missing input micrograph %s" % micFn # Update _params dictionary self._params['micFn'] = micFnMrc self._params['micDir'] = micDir self._params['ctffindOut'] = self._getCtfOutPath(micDir) self._params['ctffindPSD'] = self._getPsdPath(micDir) except Exception, ex: print >> sys.stderr, "Some error happened: %s" % ex import traceback traceback.print_exc()
def exportParticlesStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. """ imgSet = self.inputParticles.get() self._stackType = self.stackType.get() self._ih = em.ImageHandler() self._stackDict = {} particlesPath = self._getPath('Particles') pwutils.cleanPath(particlesPath) pwutils.makePath(particlesPath) alignType = imgSet.getAlignment() if self.useAlignment else em.ALIGN_NONE # Create links to binary files and write the relion .star file writeSetOfParticles(imgSet, self._getPath("particles.star"), outputDir=self._getExtraPath(), alignType=alignType, postprocessImageRow=self._postprocessImageRow, fillMagnification=True) pwutils.prettyDict(self._stackDict)
def testAssignCTF(self): """ Test the particle extraction after importing another SetOfMicraphs with a different micName but same ids. We will use assign-ctf protocol and extract from the newly imported mics with the assigned CTF. For the other mics, we will just create symbolic links. """ # Create the links with a different micrograph name micsPath = self.proj.getPath('otherMicrographs') pwutils.makePath(micsPath) for i in [6, 7, 8]: micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i) micLink = join(micsPath, basename(micPath).replace('.mrc', '_DW.mrc')) pwutils.createAbsLink(micPath, micLink) protImportDW = self.proj.copyProtocol(self.protImport) protImportDW.setObjLabel('import -mics DW') protImportDW.filesPath.set(micsPath) protImportDW.filesPattern.set('*_DW.mrc') self.launchProtocol(protImportDW) protAssignCTF = self.newProtocol(ProtCTFAssign) protAssignCTF.inputSet.set(protImportDW.outputMicrographs) protAssignCTF.inputCTF.set(self.protCTF.outputCTF) self.launchProtocol(protAssignCTF) protExtract = self.newProtocol(XmippProtExtractParticles, boxSize=183, downsampleType=OTHER, downFactor=3.0,doFlip=False) protExtract.inputCoordinates.set(self.protPP.outputCoordinates) protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs) protExtract.ctfRelations.set(self.protCTF.outputCTF) protExtract.setObjLabel("extract-other (DW mics)") self.launchProtocol(protExtract) inputCoords = protExtract.inputCoordinates.get() outputParts = protExtract.outputParticles
def _pickMicrograph(self, mic, args): # Prepare mic folder and convert if needed micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) ih = ImageHandler() # If needed convert micrograph to mrc format, otherwise link it if pwutils.getExt(micName) != ".mrc": fnMicBase = pwutils.replaceBaseExt(micName, 'mrc') inputMic = os.path.join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = os.path.join(micDir, os.path.basename(micName)) pwutils.createLink(micName, inputMic) # Program to execute and it arguments program = "ApDogPicker.py" outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt")) args += " --image=%s --outfile=%s" % (inputMic, outputFile) self.runJob(program, args)
def runTests(self, tests): self.testCount = 0 if self.log: self.testsDir = join(os.environ['SCIPION_USER_DATA'], 'Tests', self.log) pwutils.cleanPath(self.testsDir) pwutils.makePath(self.testsDir) self.testLog = join(self.testsDir, 'tests.html') self.testTimer = pwutils.Timer() self.testTimer.tic() self.headerPrefix = '<h3>Test results (%s) Duration: ' % pwutils.prettyTime() f = open(self.testLog, 'w') f.write("""<!DOCTYPE html> <html> <body> """) f.write(self.headerPrefix + '</h3>') f.write(""" <table style="width:100%" border="1"> <tr> <th>#</th> <th>Command</th> <th>Time</th> <th>Result</th> <th>Log file</th> </tr> <!-- LAST_ROW --> </table> </body> </html>""") f.close() self._visitTests(tests, self._runNewItem) if self.log: print "\n\nOpen results in your browser: \nfile:///%s" % self.testLog
""" % error sys.exit(1) n = len(sys.argv) if n < 2 or n > 3: usage("Incorrect number of input parameters") jsonFn = os.path.abspath(sys.argv[1]) now = datetime.now() tempSpace = "editor-%s" % now.strftime('%Y%m%d-%H%M%S') customUserData = os.path.join(os.environ['SCIPION_USER_DATA'], 'tmp', tempSpace) pwutils.makePath(os.path.join(customUserData, 'projects')) print "Loading projects from:\n", customUserData # Create a new project manager = Manager(SCIPION_USER_DATA=customUserData) projName = os.path.basename(jsonFn) proj = manager.createProject(projName) projPath = manager.getProjectPath(projName) proj.loadProtocols(jsonFn) class EditorProjectWindow(ProjectWindow): def close(self, e=None): try: print "Writing protocols to: ", jsonFn
def createReportDir(self): self.reportDir = os.path.abspath(self._getExtraPath(self.getProject().getShortName())) self.reportPath = os.path.join(self.reportDir, 'index.html') # create report dir pwutils.makePath(self.reportDir)
sys.exit(1) n = len(sys.argv) if n < 2 or n > 3: usage("Incorrect number of input parameters") pathToProj = os.path.abspath(sys.argv[1]) now = datetime.now() tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S') customUserData = os.path.join(os.environ['SCIPION_USER_DATA'], 'tmp', tempSpace) projectsDir = os.path.join(customUserData, 'projects') pwutils.makePath(projectsDir) print "Loading projects from:\n", projectsDir projName = os.path.basename(pathToProj) pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName)) # Create a new project manager = Manager(SCIPION_USER_DATA=customUserData) proj = manager.loadProject(projName) projPath = manager.getProjectPath(projName) class EditorProjectWindow(ProjectWindow): def close(self, e=None): try:
def convertInputStep(self): inputParticles = self.inputParticles.get() firstCoord = inputParticles.getFirstItem().getCoordinate() self.hasMicName = firstCoord.getMicName() is not None inputMics = self._getMicrographs() self.alignType = inputParticles.getAlignment() self.downFactor = self.ctfDownFactor.get() # create a tmp set for matching mics self.matchingMics = self._createSetOfMicrographs(suffix='_tmp') self.matchingMics.copyInfo(inputMics) if self.downFactor != 1.: self.matchingMics.setDownsample(self.downFactor) # create a tmp set for coords coords = self._createSetOfCoordinates(inputMics, suffix='_tmp') newCoord = Coordinate() self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() if self.scale != 1.0: print "Scaling coordinates by a factor *%0.2f*" % self.scale # Create the micrograph dicts micDict = {} # dict with micName or micId micBaseDict = {} # dict with micName (just basename) micKey2 = None insertedMics = {} for mic in inputMics: if self.hasMicName: micKey = mic.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!" % micKey) micDict[micKey] = mic.clone() if self.hasMicName: micBaseDict[micKey2] = mic.clone() # match the mic from coord with micDict for particle in inputParticles: coord = particle.getCoordinate() or None if coord is None: print "Skipping particle, coordinates not found" continue if self.hasMicName: micKey = coord.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = coord.getMicId() # find the mapping by micName (with or without ext) or micId mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) x, y = coord.getPosition() if self.applyShifts: shifts = getShifts(particle.getTransform(), self.alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) newCoord.setPosition(xCoor * self.scale, yCoor * self.scale) else: newCoord.setPosition(x * self.scale, y * self.scale) newCoord.setMicrograph(mic) coords.append(newCoord) if mic.getObjId() not in insertedMics: insertedMics[mic.getObjId()] = mic self.matchingMics.append(mic) ih = em.ImageHandler() # We convert matching micrographs if they are not *.mrc for mic in self.matchingMics: # Create micrograph dir micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if self.downFactor != 1.: ih.scaleFourier(micName, outMic, self.downFactor) sps = inputMics.getScannedPixelSize() * self.downFactor self._params['scannedPixelSize'] = sps else: if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(micName, outMic) # Write out coordinate files and sets writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics) coords.clear() pwutils.cleanPath(coords.getFileName()) self.matchingMics.write() self.matchingMics.close()
def refineCtfStep(self): self._defineValues() self._prepareCommand() for mic in self.matchingMics: micName = mic.getFileName() micBase = pwutils.removeBaseExt(micName) micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName)) outMic = pwutils.join(micDirTmp, pwutils.replaceBaseExt(micName, 'mrc')) micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf') micFnOut = self._getCtfOutPath(micDirTmp) micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log') micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star') # Update _params dictionary self._params['micFn'] = outMic self._params['gctfOut'] = micFnOut if self.useInputCtf and self.ctfRelations.get(): # get input CTFs from a mic ctfs = self.ctfRelations.get() micKey = mic.getMicName() if self.hasMicName else mic.getObjId() for ctf in ctfs: ctfMicName = ctf.getMicrograph().getMicName() ctfMicId = ctf.getMicrograph().getObjId() if micKey == ctfMicName or micKey == ctfMicId: # add CTF refine options self._params.update({'refine_input_ctf': 1, 'defU_init': ctf.getDefocusU(), 'defV_init': ctf.getDefocusV(), 'defA_init': ctf.getDefocusAngle(), 'B_init': self.bfactor.get() }) self._args += "--refine_input_ctf %d " % self._params['refine_input_ctf'] self._args += "--defU_init %f " % self._params['defU_init'] self._args += "--defV_init %f " % self._params['defV_init'] self._args += "--defA_init %f " % self._params['defA_init'] self._args += "--B_init %f " % self._params['B_init'] self._args += "--defU_err %f " % self.defUerr.get() self._args += "--defV_err %f " % self.defVerr.get() self._args += "--defA_err %f " % self.defAerr.get() self._args += "--B_err %f " % self.Berr.get() break # final args self._args += "--do_validation %d " % (1 if self.doValidate else 0) self._args += "%(micFn)s " self._args += "> %(gctfOut)s" try: self.runJob(self._getProgram(), self._args % self._params, env=self._getEnviron()) except: print("ERROR: Gctf has failed for micrograph %s" % outMic) # move results from tmp to extra folder micDir = self._getExtraPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) psdFile = self._getPsdPath(micDir) ctfOutFile = self._getCtfOutPath(micDir) ctffitFile = self._getCtfFitOutPath(micDir) ctflocalFile = self._getCtfLocalPath(micDir, micBase) pwutils.moveFile(micFnCtf, psdFile) pwutils.moveFile(micFnOut, ctfOutFile) pwutils.moveFile(micFnCtfFit, ctffitFile) pwutils.moveFile(micFnLocalCtf, ctflocalFile) # Let's clean the temporary micrographs pwutils.cleanPath(outMic) pwutils.cleanPath(micDirTmp) pwutils.cleanPath(self.matchingMics.getFileName()) pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))