def show(self, form):
prot = form.protocol
micSet = prot.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = prot.getProject()
micfn = micSet.getFileName()
# Prepare a temporary folder to convert some input files
# and put some of the intermediate result files
coordsDir = project.getTmpPath(micSet.getName())
pwutils.cleanPath(coordsDir)
pwutils.makePath(coordsDir)
prot.convertInputs(coordsDir)
pickerConfig = os.path.join(coordsDir, 'picker.conf')
f = open(pickerConfig, "w")
pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py')
pickCmd = prot._getPickArgs(threshold=False, workingDir=coordsDir)[0]
convertCmd = pw.join('apps', 'pw_convert.py')
args = {
"pickScript": pickScript,
"pickCmd": pickCmd,
"convertCmd": convertCmd,
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
'thresholdLow': prot.thresholdLow,
'thresholdHigh': prot.thresholdHigh,
"useGPU": prot.useGPU
}
f.write("""
parameters = thresholdLow,thresholdHigh
thresholdLow.value = %(thresholdLow)s
thresholdLow.label = Threshold Low
thresholdLow.help = Low value cut-off
thresholdHigh.value = %(thresholdHigh)s
thresholdHigh.label = Threshold High
thresholdHigh.help = High value cut-off
autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh)
convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, prot,
mode=CoordinatesObjectView.MODE_AUTOMATIC,
pickerProps=pickerConfig).show()
process.wait()
myprops = pwutils.readProperties(pickerConfig)
if myprops['applyChanges'] == 'true':
form.setVar('thresholdLow', myprops['thresholdLow.value'])
form.setVar('thresholdHigh', myprops['thresholdHigh.value'])
def convertInputStep(self, micsId, refsId):
""" This step will take of the convertions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrc' format
Mask: either converted ('.tif' format) or generated a circular one
"""
ih = em.ImageHandler()
micDir = self._getTmpPath('micrographs')
pwutils.makePath(micDir)
for mic in self.getInputMicrographs():
# Create micrograph folder
micName = mic.getFileName()
# If micrographs are in .mrc format just link it
# otherwise convert them
outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(mic, outMic)
refDir = self._getTmpPath('templates')
pwutils.makePath(refDir)
# We will always convert the templates, since
# they can be in an stack and link will not be possible sometimes
inputRefs = self.inputReferences.get()
for i, ref in enumerate(inputRefs):
outRef = join(refDir, 'ref%02d.mrc' % (i+1))
ih.convert(ref, outRef)
self.createInputMasks(inputRefs)
def _estimateCtfList(self, micList, *args, **kwargs):
""" Estimate several micrographs at once, probably a bit more
efficient. """
try:
micPath = self._getMicrographDir(micList[0])
if len(micList) > 1:
micPath += ('-%04d' % micList[-1].getObjId())
pwutils.makePath(micPath)
ih = emlib.image.ImageHandler()
for mic in micList:
micFn = mic.getFileName()
# We convert the input micrograph on demand if not in .mrc
downFactor = self.ctfDownFactor.get()
micFnMrc = os.path.join(micPath,
pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self._params['scannedPixelSize'] * downFactor
kwargs['scannedPixelSize'] = sps
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
program, args = self._gctfProgram.getCommand(**kwargs)
args += ' %s/*.mrc' % micPath
self.runJob(program, args) # , env=gctf.Plugin.getEnviron())
def _getFile(micBase, suffix):
return os.path.join(micPath, micBase + suffix)
for mic in micList:
micFn = mic.getFileName()
micBase = pwutils.removeBaseExt(micFn)
micFnMrc = _getFile(micBase, '.mrc')
# Let's clean the temporary mrc micrograph
pwutils.cleanPath(micFnMrc)
# move output from tmp to extra
micFnCtf = _getFile(micBase, self._gctfProgram.getExt())
micFnCtfLog = _getFile(micBase, '_gctf.log')
micFnCtfFit = _getFile(micBase, '_EPA.log')
micFnCtfOut = self._getPsdPath(micFn)
micFnCtfLogOut = self._getCtfOutPath(micFn)
micFnCtfFitOut = self._getCtfFitOutPath(micFn)
pwutils.moveFile(micFnCtf, micFnCtfOut)
pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
pwutils.cleanPath(micPath)
except:
print("ERROR: Gctf has failed on %s/*.mrc" % micPath)
import traceback
traceback.print_exc()
def testWriteSetOfCoordinatesWithoutFlip(self):
from collections import OrderedDict
# Define a temporary sqlite file for micrographs
fn = self.getOutputPath('convert_mics.sqlite')
mics = emobj.SetOfMicrographs(filename=fn)
# Create SetOfCoordinates data
# Define a temporary sqlite file for coordinates
fn = self.getOutputPath('convert_coordinates.sqlite')
coordSet = emobj.SetOfCoordinates(filename=fn)
coordSet.setBoxSize(60)
coordSet.setMicrographs(mics)
data = OrderedDict()
data['006'] = [(30, 30)]
data['016'] = [(40, 40)]
micList = []
for key, coords in data.items():
mic = emobj.Micrograph(self.ds.getFile('micrographs/%s.mrc' % key))
mics.append(mic)
micList.append(mic)
print("Adding mic: %s, id: %s" % (key, mic.getObjId()))
for x, y in coords:
coord = emobj.Coordinate(x=x, y=y)
coord.setMicrograph(mic)
coordSet.append(coord)
# Get boxDirectory
boxFolder = self.getOutputPath('boxFolder')
os.mkdir(boxFolder)
micFolder = self.getOutputPath('micFolder')
pwutils.makePath(micFolder)
# Invoke the write set of coordinates method
convert.writeSetOfCoordinates(boxFolder, coordSet)
convert.convertMicrographs(micList, micFolder)
# Assert output of writesetofcoordinates
for mic in micList:
boxFile = os.path.join(boxFolder,
convert.getMicIdName(mic, '.box'))
self.assertTrue(os.path.exists(boxFile),
'Missing box file: %s' % boxFile)
micFile = os.path.join(micFolder,
convert.getMicIdName(mic, '.mrc'))
self.assertTrue(os.path.exists(micFile),
'Missing box file: %s' % micFile)
# Assert coordinates in box files
fh = open(os.path.join(boxFolder, 'mic00001.box'))
box1 = fh.readline()
fh.close()
box1 = box1.split('\t')
self.assertEquals(box1[0], '0')
self.assertEquals(box1[1], '964')
def pysegPostRec(self, outStar):
# Generate output subtomo dir
outDir = self._getExtraPath(POST_REC_OUT)
makePath(outDir)
# Script called
Plugin.runPySeg(self, PYTHON, self._getCommand(outDir, outStar))
def testConvertCoords(self):
boxSize = 100
boxDir = self.getOutputPath('boxDir')
pwutils.makePath(boxDir)
def _convert(coordsIn, yFlipHeight=None):
tmpFile = os.path.join(boxDir, 'tmp.cbox')
# Write input coordinates
writer = convert.CoordBoxWriter(boxSize, yFlipHeight=yFlipHeight)
writer.open(tmpFile)
for x, y, _ in coordsIn:
writer.writeCoord(emobj.Coordinate(x=x, y=y))
writer.close()
reader = convert.CoordBoxReader(boxSize, yFlipHeight=yFlipHeight)
reader.open(tmpFile)
coordsOut = [c for c in reader.iterCoords()]
reader.close()
return coordsOut
coordsIn = [(100, 100, 0.), (100, 200, 0.), (200, 100, 0.), (200, 200, 0.)]
# Case 1: No flip
coordsOut = _convert(coordsIn)
for c1, c2 in zip(coordsIn, coordsOut):
self.assertEqual(c1, c2)
# Case 2: Flipping on Y
coordsOut = _convert(coordsIn, yFlipHeight=300)
for c1, c2 in zip(coordsIn, coordsOut):
self.assertEqual(c1, c2)
def _insertInitialSteps(self):
""" Override this function to insert some steps before the
estimate ctfs steps.
Should return a list of ids of the initial steps. """
pwutils.makePath(self._getExtraPath('DONE'))
return []
def exportData(emxDir, inputSet, ctfSet=None,
xmlFile='data.emx', binaryFile=None,
doConvert=True):
""" Export micrographs, coordinates or particles to EMX format. """
pwutils.cleanPath(emxDir)
pwutils.makePath(emxDir)
emxData = emxlib.EmxData()
micSet=None
if isinstance(inputSet, SetOfMicrographs):
_micrographsToEmx(emxData, inputSet, emxDir, ctfSet, writeData=True)
elif isinstance(inputSet, SetOfCoordinates):
micSet = inputSet.getMicrographs()
_micrographsToEmx(emxData, micSet, emxDir, ctfSet)
_particlesToEmx(emxData, inputSet, micSet, writeImages=False)
# _particlesToEmx(emxData, inputSet, None, micSet, doConvert=doConvert)
elif isinstance(inputSet, SetOfParticles):
if inputSet.hasCoordinates():
micSet = inputSet.getCoordinates().getMicrographs()
_micrographsToEmx(emxData, micSet, emxDir, writeData=False)
kwargs = {'writeImages': True}
if binaryFile is None:
kwargs['imagesPrefix'] = emxDir
else:
kwargs['imagesStack'] = join(emxDir, binaryFile)
_particlesToEmx(emxData, inputSet, micSet, **kwargs)
fnXml = join(emxDir, xmlFile)
emxData.write(fnXml)
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
#Replace extension by 'mrc' cause there are some formats that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
self.runJob("xmipp_transform_downsample",
"-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor),
env=xmipp3.getEnviron())
self._params['scannedPixelSize'] = self.inputMicrographs.get().getScannedPixelSize() * downFactor
else:
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def msaStep(self):
""" Run MSA on input particles. """
distances = ['EUCLIDIAN', 'CHISQUARE', 'MODULATION']
distance_name = distances[self.distanceType.get()]
self._params.update({
'msa_dir':
self.MSA_DIR,
'msa_distance':
distance_name,
'num_factors':
self.numberOfFactors.get(),
'num_iter':
self.numberOfIterations.get(),
'overcorrectionFactor':
self.overcorrectionFactor.get(),
'mpi_procs':
self.numberOfMpi.get()
})
msaDir = self._getPath(self.MSA_DIR)
if exists(msaDir):
pwutils.cleanPath(msaDir)
pwutils.makePath(msaDir)
self.runTemplate('msa/msa-run.b', self._params)
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
# Get current values of the properties
# micfn = os.path.join(coordsDir, 'micrographs.xmd')
# writeSetOfMicrographs(micSet, micfn)
dogpickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(dogpickerProps, "w")
args = {
"dogpicker":
os.path.join(os.environ['DOGPICKER_HOME'], "ApDogPicker.py"),
"convert":
pw.join('apps', 'pw_convert.py'),
'coordsDir':
coordsDir,
'micsSqlite':
micSet.getFileName(),
"diameter":
autopickProt.diameter,
"threshold":
autopickProt.threshold,
"apix":
micSet.getSamplingRate()
}
f.write("""
parameters = diameter,threshold
diameter.value = %(diameter)s
diameter.label = Diameter
diameter.help = some help
threshold.value = %(threshold)s
threshold.label = Threshold
threshold.help = some help
autopickCommand = %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameter) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt
convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project,
micfn,
coordsDir,
autopickProt,
pickerProps=dogpickerProps).show()
process.wait()
# Check if the wizard changes were accepted or just canceled
myprops = readProperties(dogpickerProps)
if myprops['applyChanges'] == 'true':
form.setVar('diameter', myprops['diameter.value'])
form.setVar('threshold', myprops['threshold.value'])
def runExecuteCtfGroupsStep(self, **kwargs):
makePath(self.ctfGroupDirectory)
self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory)
# printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line
if not self.doCTFCorrection:
md = xmipp.MetaData(self.selFileName)
block_name = self._getBlockFileName(ctfBlockName, 1,
self._getFileName('imageCTFpairs'))
md.write(block_name)
self._log.info("Written a single CTF group to file: '%s'" % block_name)
self.numberOfCtfGroups.set(1)
else:
self._log.info(
'*********************************************************************'
)
self._log.info('* Make CTF groups')
# remove all entries not present in sel file by
# join between selfile and metadatafile
mdCtfData = xmipp.MetaData()
mdCtfData.read(self.ctfDatName)
mdSel = xmipp.MetaData()
mdSel.read(self.selFileName)
mdCtfData.intersection(mdSel, xmipp.MDL_IMAGE)
tmpCtfDat = self.ctfDatName
mdCtfData.write(tmpCtfDat)
args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s'
args += ' --sampling_rate %(sampling)s'
params = {
'tmpCtfDat': tmpCtfDat,
'ctffile': self._getFileName('ctfGroupBase') + ':stk',
'wiener': self.wienerConstant.get(),
'pad': self.paddingFactor.get(),
'sampling': self.resolSam
}
if self.inputParticles.get().isPhaseFlipped():
args += ' --phase_flipped '
if self.doAutoCTFGroup:
args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s'
params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get()
params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get()
else:
if exists(self.setOfDefocus.get()):
args += ' --split %(setOfDefocus)s'
params['setOfDefocus'] = self.setOfDefocus.get()
self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs)
auxMD = xmipp.MetaData("numberGroups@" +
self._getFileName('cTFGroupSummary'))
self.numberOfCtfGroups.set(
auxMD.getValue(xmipp.MDL_COUNT, auxMD.firstObject()))
self._store(self.numberOfCtfGroups)
def writeCtfStarStep(self):
inputCTF = self.inputCTF.get()
if self.micrographSource == 0: # same as CTF estimation
ctfMicSet = inputCTF.getMicrographs()
else:
ctfMicSet = self.inputMicrographs.get()
micSet = SetOfMicrographs(filename=':memory:')
psd = inputCTF.getFirstItem().getPsdFile()
hasPsd = psd and os.path.exists(psd)
if hasPsd:
psdPath = self._getPath('PSD')
pwutils.makePath(psdPath)
print "Writing PSD files to %s" % psdPath
for ctf in inputCTF:
# Get the corresponding micrograph
mic = ctfMicSet[ctf.getObjId()]
if mic is None:
print("Skipping CTF id: %s, it is missing from input "
"micrographs. " % ctf.getObjId())
continue
micFn = mic.getFileName()
if not os.path.exists(micFn):
print "Skipping micrograph %s, it does not exists. " % micFn
continue
mic2 = mic.clone()
mic2.setCTF(ctf)
if hasPsd:
psdFile = ctf.getPsdFile()
newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
if not os.path.exists(psdFile):
print "PSD file %s does not exits" % psdFile
print "Skipping micrograph %s" % micFn
continue
pwutils.copyFile(psdFile, newPsdFile)
ctf.setPsdFile(newPsdFile)
micSet.append(mic2)
starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
print "Writing set: %s" % inputCTF
print " to: %s" % starFile
acq = ctfMicSet.getAcquisition()
self.samplingRate = ctfMicSet.getSamplingRate()
mag = acq.getMagnification()
self.detectorPixelSize = 1e-4 * self.samplingRate * mag
writeSetOfMicrographs(micSet, starFile,
preprocessImageRow=self.preprocessMicrograph)
# Let's create a link from the project roots to facilitate the import
# of the star file into a Relion project
pwutils.createLink(starFile, os.path.basename(starFile))
def show(self, form):
prot = form.protocol
micSet = prot.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = prot.getProject()
micfn = micSet.getFileName()
# Prepare a temporary folder to convert some input files
# and put some of the intermediate result files
coordsDir = project.getTmpPath(micSet.getName())
pwutils.cleanPath(coordsDir)
pwutils.makePath(coordsDir)
prot.convertInputs(coordsDir)
pickerConfig = os.path.join(coordsDir, 'picker.conf')
f = open(pickerConfig, "w")
pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py')
pickCmd = prot.getArgs(threshold=False, workingDir=coordsDir)
convertCmd = pw.join('apps', 'pw_convert.py')
args = {
"pickScript": pickScript,
"pickCmd": pickCmd,
"convertCmd": convertCmd,
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
'thresholdLow': prot.thresholdLow,
'thresholdHigh': prot.thresholdHigh,
"useGPU": prot.useGPU
}
f.write("""
parameters = thresholdLow,thresholdHigh
thresholdLow.value = %(thresholdLow)s
thresholdLow.label = Threshold Low
thresholdLow.help = Low value cut-off
thresholdHigh.value = %(thresholdHigh)s
thresholdHigh.label = Threshold High
thresholdHigh.help = High value cut-off
autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh)
convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, prot,
pickerProps=pickerConfig).show()
process.wait()
myprops = pwutils.readProperties(pickerConfig)
if myprops['applyChanges'] == 'true':
form.setVar('thresholdLow', myprops['thresholdLow.value'])
form.setVar('thresholdHigh', myprops['thresholdHigh.value'])
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
micPath = micSet.getFileName()
convert.writeSetOfCoordinatesXmipp(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def _convertVolumes(self):
ih = ImageHandler()
makePath(self._getExtraPath('input'))
inputVols = self.inputVolumes.get()
for vol in inputVols:
map = ih.read(vol)
outputFn = self._getOutputFn(vol.getObjId())
map.write(outputFn)
def _createFolder(self, p):
if os.path.exists(p):
raise Exception("Path '%s' already exists.\n" % p)
# Create the project path
sys.stdout.write("Creating path '%s' ... " % p)
pwutils.makePath(p)
os.chmod(p, 0o0775)
sys.stdout.write("DONE\n")
def initializeParams(self):
self.finished = False
self.insertedDict = {}
self.initializeRejDict()
self.setSecondaryAttributes()
self.ctfFn1 = self.inputCTF.get().getFileName()
self.allCtf1 = {}
pwutils.makePath(self._getExtraPath('DONE'))
def _processMovie(self, movie):
movieFolder = self._getOutputMovieFolder(movie)
inputStar = os.path.join(movieFolder,
'%s_input.star' % self._getMovieRoot(movie))
pwutils.makePath(os.path.join(movieFolder, 'output'))
og = OpticsGroups.fromImages(self.inputMovies.get())
writer = convert.createWriter(optics=og)
# Let's use only the basename, since we will launch the command
# from the movieFolder
movie.setFileName(os.path.basename(movie.getFileName()))
writer.writeSetOfMovies([movie], inputStar)
# The program will run in the movie folder, so let's put
# the input files relative to that
args = "--i %s --o output/ " % os.path.basename(inputStar)
args += "--use_own "
f0, fN = self._getRange(movie)
args += "--first_frame_sum %d --last_frame_sum %d " % (f0, fN)
args += "--bin_factor %f --bfactor %d " % (self.binFactor,
self.bfactor)
args += "--angpix %0.5f " % (movie.getSamplingRate())
args += "--patch_x %d --patch_y %d " % (self.patchX, self.patchY)
args += "--group_frames %d " % self.groupFrames
args += "--j %d " % self.numberOfThreads
inputMovies = self.inputMovies.get()
if inputMovies.getGain():
args += ' --gainref "%s" ' % inputMovies.getGain()
args += ' --gain_rot %d ' % self.gainRot
args += ' --gain_flip %d ' % self.gainFlip
if self.IS_GT30():
if self.defectFile.get():
args += ' --defect_file "%s" ' % self.defectFile.get()
if self._savePsSum():
args += ' --grouping_for_ps %d ' % self._calcPsDose()
if self.doDW:
args += "--dose_weighting "
preExp, dose = self._getCorrectedDose(self.inputMovies.get())
args += "--dose_per_frame %f " % dose
args += "--preexposure %f " % preExp
if self.saveNonDW:
args += " --save_noDW "
if self.extraParams.hasValue():
args += " " + self.extraParams.get()
try:
self.runJob(self._getProgram(), args, cwd=movieFolder)
self._computeExtra(movie)
self._moveFiles(movie)
except:
print("ERROR: processing movie: ", movie.getFileName())
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(
self._getPath(inputFit),
self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(
self._getPath(
'movie_particles_shiny_%sclass001_unfil.mrc' %
half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
half))
self._renameFiles('movie_particles_shiny_post*',
'movie_particles_')
self._renameFiles('movie_particles_shiny*',
'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(
mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def runCase(self, args, mpi=0, changeDir=False,
preruns=None, postruns=None, validate=None,
outputs=None, random=False):
# Retrieve the correct case number from the test name id
# We asumme here that 'test_caseXXX' should be in the name
caseId = unittest.TestCase.id(self)
if not 'test_case' in caseId:
raise Exception("'test_case' string should be in the test function name followed by a number")
_counter = int(caseId.split('test_case')[1])
self._testDir = self.dataset.getPath()
self.outputDir = os.path.join('tmpLink', '%s_%02d' % (self.program, _counter))
self.outputDirAbs = os.path.join(self._testDir, self.outputDir)
self.goldDir = os.path.join(self._testDir, 'gold', '%s_%02d' % (self.program, _counter))
# Clean and create the program output folder if not exists
pwutils.cleanPath(self.outputDirAbs)
pwutils.makePath(self.outputDirAbs)
# Change to tests root folder (self._testDir)
cwd = os.getcwd()
os.chdir(self._testDir)
if preruns:
self._runCommands(preruns, 'preruns')
if mpi:
cmd = "mpirun -np %d `which %s`" % (mpi, self.program)
else:
cmd = self.program
args = self._parseArgs(args)
if changeDir:
cmd = "cd %s ; %s %s > stdout.txt 2> stderr.txt" % (self.outputDir, cmd, args)
else:
cmd = "%s %s > %s/stdout.txt 2> %s/stderr.txt" % (cmd, args, self.outputDir, self.outputDir)
print " Command: "
print " ", pwutils.green(cmd)
#run the test itself
command = Command(cmd, env=self.env)
self._command = command
try:
command.run(timeout=self._timeout)
except KeyboardInterrupt:
command.terminate()
if postruns:
self._runCommands(postruns, 'postruns')
if outputs:
self._checkOutputs(outputs,random)
if validate:
validate()
os.chdir(cwd)
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = micSet.getFileName()
xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def exportCoordsStep(self, coordsId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
"""
pwutils.cleanPath(self._getExportPath())
pwutils.makePath(self._getExportPath())
convert.writeSetOfCoordinates(self._getExportPath(), self.getCoords(),
self._getMicPos)
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = micSet.getFileName()
xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def writeCtfStarStep(self):
inputCTF = self.inputCTF.get()
if self.micrographSource == 0: # same as CTF estimation
ctfMicSet = inputCTF.getMicrographs()
else:
ctfMicSet = self.inputMicrographs.get()
micSet = SetOfMicrographs(filename=':memory:')
psd = inputCTF.getFirstItem().getPsdFile()
hasPsd = psd and os.path.exists(psd)
if hasPsd:
psdPath = self._getPath('PSD')
pwutils.makePath(psdPath)
print "Writing PSD files to %s" % psdPath
for ctf in inputCTF:
# Get the corresponding micrograph
mic = ctfMicSet[ctf.getObjId()]
if mic is None:
print(
"Skipping CTF id: %s, it is missing from input "
"micrographs. " % ctf.getObjId())
continue
micFn = mic.getFileName()
if not os.path.exists(micFn):
print "Skipping micrograph %s, it does not exists. " % micFn
continue
mic2 = mic.clone()
mic2.setCTF(ctf)
if hasPsd:
psdFile = ctf.getPsdFile()
newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
if not os.path.exists(psdFile):
print "PSD file %s does not exits" % psdFile
print "Skipping micrograph %s" % micFn
continue
pwutils.copyFile(psdFile, newPsdFile)
ctf.setPsdFile(newPsdFile)
micSet.append(mic2)
starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
print "Writing set: %s" % inputCTF
print " to: %s" % starFile
writeSetOfMicrographs(micSet,
starFile,
preprocessImageRow=self.preprocessMicrograph)
# Let's create a link from the project roots to facilitate the import
# of the star file into a Relion project
pwutils.createLink(starFile, os.path.basename(starFile))
def pysegPicking(self):
# Generate output dir
outDir = self._getExtraPath()
makePath(outDir)
# Generate slices xml
self._createPickingXmlFile(Plugin.getHome(PICKING_SLICES), outDir)
# Script called
Plugin.runPySeg(self, PYTHON, self._getPickingCommand(outDir))
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = os.path.join(coordPath, 'micrographs.xmd')
xmipp3.writeSetOfMicrographs(micSet, micPath)
xmipp3.writeSetOfCoordinates(coordPath, coordSet)
return micPath, coordPath
def convertInputStep(self, protId):
pwutils.makePath(self._getInputPath())
protRef = self.protRefine.get()
vols = protRef.getFinalVolumes() # final, half1, half2
ih = ImageHandler()
vols.append(self.solventMask.get())
for vol, key in zip(vols, ['finalVolume', 'half1', 'half2', 'mask']):
ih.convert(vol, self._getFileName(key))
def _moveCoordsToInfo(self, tomo):
fnCoor = '*%s_info.json' % pwutils.removeBaseExt(tomo.getFileName().split("__")[0])
pattern = os.path.join(self.path, fnCoor)
files = glob.glob(pattern)
if files:
infoDir = pwutils.join(os.path.abspath(self.path), 'info')
pathCoor = os.path.join(infoDir, os.path.basename(files[0]))
pwutils.makePath(infoDir)
copyFile(files[0], pathCoor)
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
pickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(pickerProps, "w")
params = ['boxSize', 'lowerThreshold', 'higherThreshold', 'gaussWidth']
args = {
"params": ','.join(params),
"preprocess" : os.path.join(os.environ['SCIPION_HOME'], "scipion sxprocess.py"),
"picker" : os.path.join(os.environ['SCIPION_HOME'], "scipion e2boxer.py"),
"convert" : os.path.join(os.environ['SCIPION_HOME'], os.path.join('pyworkflow','apps', 'pw_convert.py')),
'coordsDir':coordsDir,
'micsSqlite': micSet.getFileName(),
"boxSize": autopickProt.boxSize,
"lowerThreshold": autopickProt.lowerThreshold,
"higherThreshold": autopickProt.higherThreshold,
"gaussWidth": autopickProt.gaussWidth,
"extraParams":autopickProt.extraParams
}
f.write("""
parameters = %(params)s
boxSize.value = %(boxSize)s
boxSize.label = Box Size
boxSize.help = some help
lowerThreshold.value = %(lowerThreshold)s
lowerThreshold.label = Lower Threshold
lowerThreshold.help = some help
higherThreshold.help = some help
higherThreshold.value = %(higherThreshold)s
higherThreshold.label = Higher Threshold
gaussWidth.help = some help
gaussWidth.value = %(gaussWidth)s
gaussWidth.label = Gauss Width
runDir = %(coordsDir)s
preprocessCommand = %(preprocess)s demoparms --makedb=thr_low=%%(lowerThreshold):thr_hi=%%(higherThreshold):boxsize=%%(boxSize):gauss_width=%%(gaussWidth):%(extraParams)s
autopickCommand = %(picker)s --gauss_autoboxer=demoparms --write_dbbox --boxsize=%%(boxSize) --norm=normalize.ramp.normvar %%(micrograph)
convertCommand = %(convert)s --coordinates --from eman2 --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt, pickerProps=pickerProps).show()
process.wait()
myprops = readProperties(pickerProps)
for param in params:
form.setVar(param, myprops[param + '.value'])
def convertImagesStep(self):
partSet = self._getInputParticles()
partAlign = partSet.getAlignment()
storePath = self._getExtraPath("particles")
makePath(storePath)
writeSetOfParticles(partSet, storePath, alignType=partAlign)
if self.useInputBispec and self.inputBispec is not None:
print(
"Skipping CTF estimation since input bispectra were provided")
self.skipctf.set(True)
if not self.skipctf:
program = eman2.Plugin.getProgram('e2ctf.py')
acq = partSet.getAcquisition()
args = " --voltage %d" % acq.getVoltage()
args += " --cs %f" % acq.getSphericalAberration()
args += " --ac %f" % (100 * acq.getAmplitudeContrast())
args += " --threads=%d" % self.numberOfThreads.get()
if not partSet.isPhaseFlipped():
args += " --phaseflip"
args += " --computesf --apix %f" % partSet.getSamplingRate()
args += " --allparticles --autofit --curdefocusfix --storeparm -v 8"
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
program = eman2.Plugin.getProgram('e2buildsets.py')
args = " --setname=inputSet --allparticles --minhisnr=-1"
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
if self.useInputBispec:
prot = self.inputBispec.get()
prot._createFilenameTemplates()
bispec = prot.outputParticles_flip_bispec
if not bispec.getSize() == partSet.getSize():
raise Exception(
'Input particles and bispectra sets have different size!')
# link bispec hdf files and lst file
pattern = prot._getExtraPath('particles/*__ctf_flip_bispec.hdf')
print("\nLinking bispectra input files...")
for fn in sorted(glob(pattern)):
newFn = join(self._getExtraPath('particles'), basename(fn))
createLink(fn, newFn)
print(" %s -> %s" % (fn, newFn))
lstFn = prot._getFileName('partSetFlipBispec')
newLstFn = self._getFileName('partBispecSet')
createLink(lstFn, newLstFn)
def _insertAllSteps(self):
pwutils.makePath(self._getExtraPath('inputCoords'))
pwutils.makePath(self._getExtraPath('outputCoords'))
coordinates = self.inputCoordinates.get()
self.tomos = coordinates.getPrecedents()
self._insertFunctionStep('computeParams', coordinates)
if self.outliers.get():
self._insertFunctionStep('detectOutliers')
if self.carbon.get():
self._insertFunctionStep('detectCarbonCloseness', coordinates)
self._insertFunctionStep('createOutputStep', coordinates)
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print('must specify input micrographs')
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
# Get current values of the properties
# micfn = os.path.join(coordsDir, 'micrographs.xmd')
# writeSetOfMicrographs(micSet, micfn)
dogpickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(dogpickerProps, "w")
args = {
"dogpicker": os.path.join(Plugin.getHome(), "ApDogPicker.py"),
"convert": 'emconvert',
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
"diameter": autopickProt.diameter,
"threshold": autopickProt.threshold,
"apix": micSet.getSamplingRate()
}
f.write("""
parameters = diameterA,threshold
diameterA.value = %(diameter)s
diameterA.label = Diameter in A
diameterA.help = Sampling rate is %(apix)s
threshold.value = %(threshold)s
threshold.label = Threshold
threshold.help = Threshold in standard deviations above the mean
autopickCommand = python2 %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameterA) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt
convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(
project,
micfn,
coordsDir,
autopickProt,
mode=CoordinatesObjectView.MODE_AUTOMATIC,
pickerProps=dogpickerProps).show()
process.wait()
# Check if the wizard changes were accepted or just canceled
myprops = readProperties(dogpickerProps)
if myprops['applyChanges'] == 'true':
form.setVar('diameter', myprops['diameterA.value'])
form.setVar('threshold', myprops['threshold.value'])
def convertInputStep(self):
""" This step will take of the conversions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrcs' format
"""
micDir = self.getMicrographsDir() # put output and mics in extra dir
pwutils.makePath(micDir)
# We will always convert the templates to mrcs stack
self.convertReferences(self._getExtraPath('references.mrcs'))
# Convert input coords for exclusive picking
self.convertCoordinates(micDir)
def _convertInputStep(self):
pwutils.makePath(self._getExtraPath('DONE'))
movs = self.inputMovies.get()
# Convert gain
gain = movs.getGain()
movs.setGain(self.__convertCorrectionImage(gain))
# Convert dark
dark = movs.getDark()
movs.setDark(self.__convertCorrectionImage(dark))
def convertInputStep(self):
""" This step will take of the conversions from the inputs.
Micrographs: they will be linked if are in '.mrc' format,
converted otherwise.
References: will always be converted to '.mrcs' format
"""
pwutils.makePath(self.getMicrographsDir()) # put output and mics in extra dir
# We will always convert the templates to mrcs stack
self.convertReferences(self._getReferencesFn())
# Write defects star file if necessary
if self.exclusive and self.inputDefects.get():
writeDefectsFile(self.inputDefects.get(), self._getDefectsFn())
def __init__(self, srcPath, dstPath, copyFiles=False):
self._src = srcPath
self._dst = dstPath
self._processed = set()
if copyFiles:
pwutils.makePath(dstPath)
self.transform = self._copyFile
else:
pwutils.makePath(os.path.dirname(srcPath))
pwutils.createLink(srcPath, dstPath)
self.transform = self._addPrefix
def convertInputStep(self, micsId, refsId):
pwutils.makePath(
self._getExtraPath('DONE')) # Required to report finished
inputRefs = self.getInputReferences()
if self.useInputReferences():
relion.convert.writeReferences(inputRefs,
self._getPath('reference_2d'),
useBasename=True)
else:
ImageHandler().convert(inputRefs,
self._getPath('reference_3d.mrc'))
def runExecuteCtfGroupsStep(self, **kwargs):
makePath(self.ctfGroupDirectory)
self._log.info("Created CTF directory: '%s'" % self.ctfGroupDirectory)
# printLog("executeCtfGroups01"+ CTFDatName, _log) FIXME: print in log this line
if not self.doCTFCorrection:
md = xmipp.MetaData(self.selFileName)
block_name = self._getBlockFileName(ctfBlockName, 1, self._getFileName('imageCTFpairs'))
md.write(block_name)
self._log.info("Written a single CTF group to file: '%s'" % block_name)
self.numberOfCtfGroups.set(1)
else:
self._log.info('*********************************************************************')
self._log.info('* Make CTF groups')
# remove all entries not present in sel file by
# join between selfile and metadatafile
mdCtfData = xmipp.MetaData()
mdCtfData.read(self.ctfDatName)
mdSel = xmipp.MetaData();
mdSel.read(self.selFileName)
mdCtfData.intersection(mdSel, xmipp.MDL_IMAGE)
tmpCtfDat = self.ctfDatName
mdCtfData.write(tmpCtfDat)
args = ' --ctfdat %(tmpCtfDat)s -o %(ctffile)s --wiener --wc %(wiener)s --pad %(pad)s'
args += ' --sampling_rate %(sampling)s'
params = {'tmpCtfDat' : tmpCtfDat,
'ctffile' : self._getFileName('ctfGroupBase') + ':stk',
'wiener' : self.wienerConstant.get(),
'pad' : self.paddingFactor.get(),
'sampling' : self.resolSam
}
if self.inputParticles.get().isPhaseFlipped():
args += ' --phase_flipped '
if self.doAutoCTFGroup:
args += ' --error %(ctfGroupMaxDiff)s --resol %(ctfGroupMaxResol)s'
params['ctfGroupMaxDiff'] = self.ctfGroupMaxDiff.get()
params['ctfGroupMaxResol'] = self.ctfGroupMaxResol.get()
else:
if exists(self.setOfDefocus.get()):
args += ' --split %(setOfDefocus)s'
params['setOfDefocus'] = self.setOfDefocus.get()
self.runJob("xmipp_ctf_group", args % params, numberOfMpi=1, **kwargs)
auxMD = xmipp.MetaData("numberGroups@" + self._getFileName('cTFGroupSummary'))
self.numberOfCtfGroups.set(auxMD.getValue(xmipp.MDL_COUNT, auxMD.firstObject()))
self._store(self.numberOfCtfGroups)
def convertInputStep(self):
printLicense()
self.info('Wrinting pytom xml file: ' + self.volXml)
volsDir = self._getExtraPath('inputVolumes')
pwutils.makePath(volsDir)
writeSetOfVolumes(self.inputVolumes.get(), self.volXml, volsDir)
if self.provideReferences:
ih = em.ImageHandler()
self.info('Converting input references')
for r, vol in enumerate(self.inputReferences.get()):
refFn = self._getExtraPath('reference_%02d.mrc' % (r+1))
ih.convert(vol, refFn)
def writeSetOfCoordinates3D(self):
info_path = self._getExtraPath('info')
pwutils.makePath(info_path)
coords = self.inputCoordinates.get()
tomos = coords.getPrecedents()
tltSeries = recoverTSFromObj(coords, self)
self.json_files, self.tomo_files = jsonFilesFromSet(tomos, info_path)
_ = setCoords3D2Jsons(self.json_files, coords)
_ = tltParams2Json(self.json_files, tltSeries, mode="a")
if self.inputCTF.get() is not None:
_ = ctf2Json(self.json_files, self.inputCTF.get(), mode='a')
def _insertAllSteps(self):
numTomo = 0
makePath(self._getPredictConfDir())
# Insert processing steps
for evenTomo, oddTomo in zip(self.even.get(), self.odd.get()):
self._insertFunctionStep(self.preparePredictStep,
evenTomo.getFileName(),
oddTomo.getFileName(), numTomo)
self._insertFunctionStep(self.predictStep, numTomo)
numTomo += 1
self._insertFunctionStep(self.createOutputStep)
def convertInputStep(self):
printLicense()
self.info('Writing pytom xml file: ' + self.volXml)
volsDir = self._getExtraPath('inputVolumes')
pwutils.makePath(volsDir)
writeSetOfVolumes(self.inputVolumes.get(), self.volXml, volsDir)
ih = em.ImageHandler()
self.info('Converting input reference to: ' + self.refVol)
ih.convert(self.inputReference.get(), self.refVol)
self.info('Converting input mask to: ' + self.maskVol)
ih.convert(self.alignmentMask.get(), self.maskVol)
def setUp(self):
"""Setup actions for the html report: create directories to store
thumbnails, check if thumbnails are already generated in the
alignment protocol.
"""
# make report folders
pwutils.makePath(join(self.reportDir, MIC_THUMBS),
join(self.reportDir, PSD_THUMBS),
join(self.reportDir, SHIFT_THUMBS))
# check if align protocol already has thumbnails
if (hasattr(self.alignProtocol, 'doComputeMicThumbnail')
and self.alignProtocol._doComputeMicThumbnail()):
self.micThumbSymlinks = True
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
# Get current values of the properties
# micfn = os.path.join(coordsDir, 'micrographs.xmd')
# writeSetOfMicrographs(micSet, micfn)
dogpickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(dogpickerProps, "w")
args = {
"dogpicker" : os.path.join(os.environ['DOGPICKER_HOME'], "ApDogPicker.py"),
"convert" : pw.join('apps', 'pw_convert.py'),
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
"diameter": autopickProt.diameter,
"threshold": autopickProt.threshold,
"apix": micSet.getSamplingRate()
}
f.write("""
parameters = diameter,threshold
diameter.value = %(diameter)s
diameter.label = Diameter
diameter.help = some help
threshold.value = %(threshold)s
threshold.label = Threshold
threshold.help = some help
autopickCommand = %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameter) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt
convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt,
mode=CoordinatesObjectView.MODE_AUTOMATIC,
pickerProps=dogpickerProps).show()
process.wait()
# Check if the wizard changes were accepted or just canceled
myprops = readProperties(dogpickerProps)
if myprops['applyChanges'] == 'true':
form.setVar('diameter', myprops['diameter.value'])
form.setVar('threshold', myprops['threshold.value'])
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def testImportFromCTffind4Conflict(self):
micsPath = os.path.abspath(self.proj.getTmpPath('micrographs'))
ctfsPath = os.path.abspath(self.proj.getTmpPath('ctfs'))
pwutils.makePath(micsPath)
pwutils.makePath(ctfsPath)
micDict = {'BPV_1386': 'mic1',
'BPV_1387': 'mic10',
'BPV_1388': 'mic2'}
pattern = 'micrographs/%s.mrc'
for k, v in micDict.iteritems():
# Create a micrograph link with a given naming convention
pwutils.createAbsLink(self.dsXmipp.getFile(pattern % k),
self.proj.getTmpPath(pattern % v))
pwutils.createAbsLink(self.dsGrigorieff.getFile('ctffind4/%s' % k),
self.proj.getTmpPath('ctfs/%s' % v))
protCTF = self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
filesPath=ctfsPath,
filesPattern='mic*/*txt')
protCTF.inputMicrographs.set(self.protImport.outputMicrographs)
protCTF.setObjLabel('import from ctffind3 - empty')
self.launchProtocol(protCTF)
# FIXME: After the execution of the above case, some empty sets
# are created, if there is no matching, there should not be any output
# Let's import now the correct names, but with the problematic matching
protImport2 = self.newProtocol(ProtImportMicrographs,
filesPath=micsPath,
filesPattern='mic*mrc',
samplingRate=1.237, voltage=300)
self.launchProtocol(protImport2)
protCTF2 = self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_GRIGORIEFF,
filesPath=ctfsPath,
filesPattern='mic*/*txt')
protCTF2.inputMicrographs.set(protImport2.outputMicrographs)
protCTF2.setObjLabel('import from ctffind3 - match')
self.launchProtocol(protCTF2)
# The whole set (3 items) should find its corresponding CTF
self.assertEqual(protCTF2.outputCTF.getSize(), 3)
def _writeRefinementScripts(self):
""" Write the needed scripts to run refinement
and substitute some values.
"""
refPath = self._getExtraPath('Refinement')
pwutils.makePath(refPath)
def path(p):
""" Escape path with '' and add ../ """
return "'%s'" % join('..', p)
def script(name, paramsDict={}):
outputScript=join(refPath, name)
writeScript(getScript('projmatch', 'Refinement', name), outputScript, paramsDict)
nIter = self.numberOfIterations.get()
def getListStr(valueStr):
return "'%s'" % ','.join(pwutils.getListFromValues(valueStr, nIter))
diam = int(self.radius.get() * 2 * self.inputParticles.get().getSamplingRate())
params = {'[shrange]': self.alignmentShift.get(),
'[iter-end]': self.numberOfIterations.get(),
'[diam]': diam,
'[win-frac]': self.winFrac.get(),
'[converg]': self.convergence.get(),
'[small-ang]': '1' if self.smallAngle else '0',
'[ang-steps]': getListStr(self.angSteps.get()),
'[ang-limits]': getListStr(self.angLimits.get()),
'[ang-step-sm]': '(%0.2f)' % self.angStepSm.get(),
'[theta-range]': '(%0.2f)' % self.thetaRange.get(),
'[vol_orig]': path('vol001'),
'[sel_group_orig]': path('sel_group'),
'[sel_particles_orig]': path('group{***[grp]}_selfile'),
'[group_align_orig]': path('group{***[grp]}_align'),
'[unaligned_images_orig]': path('group{***[grp]}_stack'),
}
script('refine_settings.pam', params)
for s in ['refine', 'prepare', 'grploop', 'mergegroups',
'enhance', 'endmerge', 'smangloop', 'endrefine']:
script('%s.pam' % s)
def msaStep(self):
""" Run MSA on input particles. """
distances = ['EUCLIDIAN', 'CHISQUARE', 'MODULATION']
distance_name = distances[self.distanceType.get()]
self._params.update({'msa_dir': self.MSA_DIR,
'msa_distance': distance_name,
'num_factors': self.numberOfFactors.get(),
'num_iter': self.numberOfIterations.get(),
'overcorrectionFactor': self.overcorrectionFactor.get(),
'mpi_procs': self.numberOfMpi.get()
})
msaDir = self._getPath(self.MSA_DIR)
if exists(msaDir):
pwutils.cleanPath(msaDir)
pwutils.makePath(msaDir)
self.runTemplate('msa/msa-run.b', self._params)
def createInputMasks(self, inputRefs):
""" Create the needed mask for picking.
We should either generate a circular mask,
or convert the inputs (one or just one per reference 2d)
"""
maskSchDir = self._getTmpPath('maskSch')
pwutils.makePath(maskSchDir)
ih = em.ImageHandler()
if self.maskType == MASK_CIRCULAR:
if self.maskRadius < 0: # usually -1
radius = inputRefs.getDim()[0]/2 # use half of input dim
else:
radius = self.maskRadius.get()
outMask = join(maskSchDir, 'ref01.tif')
ih.createCircularMask(radius, inputRefs.getFirstItem(), outMask)
else:
for i, mask in enumerate(self.inputMasks):
outMask = join(maskSchDir, 'ref%02d.tif' % (i+1))
ih.convert(mask.get(), outMask)
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' because there are some formats
# that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
args = "-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor)
self.runJob("xmipp_transform_downsample",
args, env=xmipp3.getEnviron())
self._params['scannedPixelSize'] = scannedPixelSize * downFactor
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
def exportParticlesStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
"""
imgSet = self.inputParticles.get()
self._stackType = self.stackType.get()
self._ih = em.ImageHandler()
self._stackDict = {}
particlesPath = self._getPath('Particles')
pwutils.cleanPath(particlesPath)
pwutils.makePath(particlesPath)
alignType = imgSet.getAlignment() if self.useAlignment else em.ALIGN_NONE
# Create links to binary files and write the relion .star file
writeSetOfParticles(imgSet, self._getPath("particles.star"),
outputDir=self._getExtraPath(),
alignType=alignType,
postprocessImageRow=self._postprocessImageRow,
fillMagnification=True)
pwutils.prettyDict(self._stackDict)
def testAssignCTF(self):
""" Test the particle extraction after importing another
SetOfMicraphs with a different micName but same ids.
We will use assign-ctf protocol and extract from the
newly imported mics with the assigned CTF.
For the other mics, we will just create symbolic links.
"""
# Create the links with a different micrograph name
micsPath = self.proj.getPath('otherMicrographs')
pwutils.makePath(micsPath)
for i in [6, 7, 8]:
micPath = self.dataset.getFile('micrographs/BPV_138%d.mrc' % i)
micLink = join(micsPath, basename(micPath).replace('.mrc', '_DW.mrc'))
pwutils.createAbsLink(micPath, micLink)
protImportDW = self.proj.copyProtocol(self.protImport)
protImportDW.setObjLabel('import -mics DW')
protImportDW.filesPath.set(micsPath)
protImportDW.filesPattern.set('*_DW.mrc')
self.launchProtocol(protImportDW)
protAssignCTF = self.newProtocol(ProtCTFAssign)
protAssignCTF.inputSet.set(protImportDW.outputMicrographs)
protAssignCTF.inputCTF.set(self.protCTF.outputCTF)
self.launchProtocol(protAssignCTF)
protExtract = self.newProtocol(XmippProtExtractParticles,
boxSize=183, downsampleType=OTHER,
downFactor=3.0,doFlip=False)
protExtract.inputCoordinates.set(self.protPP.outputCoordinates)
protExtract.inputMicrographs.set(protAssignCTF.outputMicrographs)
protExtract.ctfRelations.set(self.protCTF.outputCTF)
protExtract.setObjLabel("extract-other (DW mics)")
self.launchProtocol(protExtract)
inputCoords = protExtract.inputCoordinates.get()
outputParts = protExtract.outputParticles
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
self.runJob(program, args)
def runTests(self, tests):
self.testCount = 0
if self.log:
self.testsDir = join(os.environ['SCIPION_USER_DATA'], 'Tests', self.log)
pwutils.cleanPath(self.testsDir)
pwutils.makePath(self.testsDir)
self.testLog = join(self.testsDir, 'tests.html')
self.testTimer = pwutils.Timer()
self.testTimer.tic()
self.headerPrefix = '<h3>Test results (%s) Duration: ' % pwutils.prettyTime()
f = open(self.testLog, 'w')
f.write("""<!DOCTYPE html>
<html>
<body>
""")
f.write(self.headerPrefix + '</h3>')
f.write("""
<table style="width:100%" border="1">
<tr>
<th>#</th>
<th>Command</th>
<th>Time</th>
<th>Result</th>
<th>Log file</th>
</tr>
<!-- LAST_ROW -->
</table>
</body>
</html>""")
f.close()
self._visitTests(tests, self._runNewItem)
if self.log:
print "\n\nOpen results in your browser: \nfile:///%s" % self.testLog
""" % error
sys.exit(1)
n = len(sys.argv)
if n < 2 or n > 3:
usage("Incorrect number of input parameters")
jsonFn = os.path.abspath(sys.argv[1])
now = datetime.now()
tempSpace = "editor-%s" % now.strftime('%Y%m%d-%H%M%S')
customUserData = os.path.join(os.environ['SCIPION_USER_DATA'],
'tmp', tempSpace)
pwutils.makePath(os.path.join(customUserData, 'projects'))
print "Loading projects from:\n", customUserData
# Create a new project
manager = Manager(SCIPION_USER_DATA=customUserData)
projName = os.path.basename(jsonFn)
proj = manager.createProject(projName)
projPath = manager.getProjectPath(projName)
proj.loadProtocols(jsonFn)
class EditorProjectWindow(ProjectWindow):
def close(self, e=None):
try:
print "Writing protocols to: ", jsonFn
def createReportDir(self):
self.reportDir = os.path.abspath(self._getExtraPath(self.getProject().getShortName()))
self.reportPath = os.path.join(self.reportDir, 'index.html')
# create report dir
pwutils.makePath(self.reportDir)
sys.exit(1)
n = len(sys.argv)
if n < 2 or n > 3:
usage("Incorrect number of input parameters")
pathToProj = os.path.abspath(sys.argv[1])
now = datetime.now()
tempSpace = "loading-%s" % now.strftime('%Y%m%d-%H%M%S')
customUserData = os.path.join(os.environ['SCIPION_USER_DATA'],
'tmp', tempSpace)
projectsDir = os.path.join(customUserData, 'projects')
pwutils.makePath(projectsDir)
print "Loading projects from:\n", projectsDir
projName = os.path.basename(pathToProj)
pwutils.createAbsLink(pathToProj, os.path.join(projectsDir, projName))
# Create a new project
manager = Manager(SCIPION_USER_DATA=customUserData)
proj = manager.loadProject(projName)
projPath = manager.getProjectPath(projName)
class EditorProjectWindow(ProjectWindow):
def close(self, e=None):
try:
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale, yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()
def refineCtfStep(self):
self._defineValues()
self._prepareCommand()
for mic in self.matchingMics:
micName = mic.getFileName()
micBase = pwutils.removeBaseExt(micName)
micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
outMic = pwutils.join(micDirTmp, pwutils.replaceBaseExt(micName, 'mrc'))
micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
micFnOut = self._getCtfOutPath(micDirTmp)
micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')
# Update _params dictionary
self._params['micFn'] = outMic
self._params['gctfOut'] = micFnOut
if self.useInputCtf and self.ctfRelations.get():
# get input CTFs from a mic
ctfs = self.ctfRelations.get()
micKey = mic.getMicName() if self.hasMicName else mic.getObjId()
for ctf in ctfs:
ctfMicName = ctf.getMicrograph().getMicName()
ctfMicId = ctf.getMicrograph().getObjId()
if micKey == ctfMicName or micKey == ctfMicId:
# add CTF refine options
self._params.update({'refine_input_ctf': 1,
'defU_init': ctf.getDefocusU(),
'defV_init': ctf.getDefocusV(),
'defA_init': ctf.getDefocusAngle(),
'B_init': self.bfactor.get()
})
self._args += "--refine_input_ctf %d " % self._params['refine_input_ctf']
self._args += "--defU_init %f " % self._params['defU_init']
self._args += "--defV_init %f " % self._params['defV_init']
self._args += "--defA_init %f " % self._params['defA_init']
self._args += "--B_init %f " % self._params['B_init']
self._args += "--defU_err %f " % self.defUerr.get()
self._args += "--defV_err %f " % self.defVerr.get()
self._args += "--defA_err %f " % self.defAerr.get()
self._args += "--B_err %f " % self.Berr.get()
break
# final args
self._args += "--do_validation %d " % (1 if self.doValidate else 0)
self._args += "%(micFn)s "
self._args += "> %(gctfOut)s"
try:
self.runJob(self._getProgram(), self._args % self._params,
env=self._getEnviron())
except:
print("ERROR: Gctf has failed for micrograph %s" % outMic)
# move results from tmp to extra folder
micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
psdFile = self._getPsdPath(micDir)
ctfOutFile = self._getCtfOutPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
ctflocalFile = self._getCtfLocalPath(micDir, micBase)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnOut, ctfOutFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.moveFile(micFnLocalCtf, ctflocalFile)
# Let's clean the temporary micrographs
pwutils.cleanPath(outMic)
pwutils.cleanPath(micDirTmp)
pwutils.cleanPath(self.matchingMics.getFileName())
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
