def runProjectionMatching(self, iterN, refN, args, **kwargs):
""" Loop over all CTF groups and launch a projection matching for each one.
Note: Iterate ctf groups in reverse order to have same order as
in add_to docfiles from angular_class_average. #FIXME: check why reverse order is needed
"""
projMatchRootName = self._getFileName('projMatchRootNames', iter=iterN, ref=refN)
refname = self._getFileName('projectLibraryStk', iter=iterN, ref=refN)
numberOfCtfGroups = self.numberOfCtfGroups.get()
# ctfGroupName = self._getPath(self.ctfGroupDirectory, '%(ctfGroupRootName)s')
#remove output metadata
cleanPath(projMatchRootName)
for ctfN in reversed(list(self.allCtfGroups())):
self._log.info('CTF group: %d/%d' % (ctfN, numberOfCtfGroups))
ctfArgs = ' -i %(inputdocfile)s -o %(outputname)s --ref %(refname)s'
inputdocfile = self._getBlockFileName(ctfBlockName, ctfN, self.docFileInputAngles[iterN-1])
outputname = self._getBlockFileName(ctfBlockName, ctfN, projMatchRootName)
baseTxtFile = removeExt(refname)
neighbFile = baseTxtFile + '_sampling.xmd'
cleanPath(neighbFile)
neighbFileb = baseTxtFile + '_group' + str(ctfN).zfill(self.FILENAMENUMBERLENGTH) + '_sampling.xmd'
copyFile(neighbFileb, neighbFile)
print "copied file ", neighbFileb, "to", neighbFile
if self.doCTFCorrection and self._referenceIsCtfCorrected[iterN]:
ctfArgs += ' --ctf %s' % self._getBlockFileName('', ctfN, self._getFileName('stackCTFs'))
progArgs = ctfArgs % locals() + args
self.runJob('xmipp_angular_projection_matching', progArgs, **kwargs)
def _estimateCtfList(self, micList, *args, **kwargs):
""" Estimate several micrographs at once, probably a bit more
efficient. """
try:
micPath = self._getMicrographDir(micList[0])
if len(micList) > 1:
micPath += ('-%04d' % micList[-1].getObjId())
pwutils.makePath(micPath)
ih = emlib.image.ImageHandler()
for mic in micList:
micFn = mic.getFileName()
# We convert the input micrograph on demand if not in .mrc
downFactor = self.ctfDownFactor.get()
micFnMrc = os.path.join(micPath,
pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self._params['scannedPixelSize'] * downFactor
kwargs['scannedPixelSize'] = sps
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
program, args = self._gctfProgram.getCommand(**kwargs)
args += ' %s/*.mrc' % micPath
self.runJob(program, args) # , env=gctf.Plugin.getEnviron())
def _getFile(micBase, suffix):
return os.path.join(micPath, micBase + suffix)
for mic in micList:
micFn = mic.getFileName()
micBase = pwutils.removeBaseExt(micFn)
micFnMrc = _getFile(micBase, '.mrc')
# Let's clean the temporary mrc micrograph
pwutils.cleanPath(micFnMrc)
# move output from tmp to extra
micFnCtf = _getFile(micBase, self._gctfProgram.getExt())
micFnCtfLog = _getFile(micBase, '_gctf.log')
micFnCtfFit = _getFile(micBase, '_EPA.log')
micFnCtfOut = self._getPsdPath(micFn)
micFnCtfLogOut = self._getCtfOutPath(micFn)
micFnCtfFitOut = self._getCtfFitOutPath(micFn)
pwutils.moveFile(micFnCtf, micFnCtfOut)
pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
pwutils.cleanPath(micPath)
except:
print("ERROR: Gctf has failed on %s/*.mrc" % micPath)
import traceback
traceback.print_exc()
def exportData(emxDir, inputSet, ctfSet=None,
xmlFile='data.emx', binaryFile=None,
doConvert=True):
""" Export micrographs, coordinates or particles to EMX format. """
pwutils.cleanPath(emxDir)
pwutils.makePath(emxDir)
emxData = emxlib.EmxData()
micSet=None
if isinstance(inputSet, SetOfMicrographs):
_micrographsToEmx(emxData, inputSet, emxDir, ctfSet, writeData=True)
elif isinstance(inputSet, SetOfCoordinates):
micSet = inputSet.getMicrographs()
_micrographsToEmx(emxData, micSet, emxDir, ctfSet)
_particlesToEmx(emxData, inputSet, micSet, writeImages=False)
# _particlesToEmx(emxData, inputSet, None, micSet, doConvert=doConvert)
elif isinstance(inputSet, SetOfParticles):
if inputSet.hasCoordinates():
micSet = inputSet.getCoordinates().getMicrographs()
_micrographsToEmx(emxData, micSet, emxDir, writeData=False)
kwargs = {'writeImages': True}
if binaryFile is None:
kwargs['imagesPrefix'] = emxDir
else:
kwargs['imagesStack'] = join(emxDir, binaryFile)
_particlesToEmx(emxData, inputSet, micSet, **kwargs)
fnXml = join(emxDir, xmlFile)
emxData.write(fnXml)
def _createSetOfParts(self, nMics=10, nOptics=2, partsPerMic=10):
micSet = self._createSetOfMics(nMics, nOptics)
outputSqlite = self.getOutputPath('particles.sqlite')
cleanPath(outputSqlite)
print(">>> Writing to particles db: %s" % outputSqlite)
outputParts = SetOfParticles(filename=outputSqlite)
outputParts.setSamplingRate(1.234)
outputParts.setAcquisition(micSet.getAcquisition())
part = SetOfParticles.ITEM_TYPE()
coord = Coordinate()
for mic in micSet:
for i in range(1, partsPerMic + 1):
part.setLocation(i, mic.getFileName().replace('mrc', 'mrcs'))
coord.setPosition(x=np.random.randint(0, 1000),
y=np.random.randint(0, 1000))
coord.setMicrograph(mic)
part.setObjId(None)
part.setCoordinate(coord)
part.setAcquisition(mic.getAcquisition())
outputParts.append(part)
outputParts.write()
return outputParts
def _create_tiltseries(self, tiltSeriesClass):
setFn = self.getOutputPath('%s.sqlite' % tiltSeriesClass.__name__)
pwutils.cleanPath(setFn)
testSet = tiltSeriesClass(filename=setFn)
for tsId in ['TS01', 'TS02', 'TS03']:
tsObj = testSet.ITEM_TYPE(tsId=tsId)
testSet.append(tsObj)
for i, a in enumerate(range(-60, 60, 5)):
tsFn = '%s_%02d_%s.mrc' % (tsId, i, a)
tim = tsObj.ITEM_TYPE(location=tsFn,
acquisitionOrder=i,
tiltAngle=a)
tsObj.append(tim)
testSet.write()
testSet.close()
testSet2 = SetOfTiltSeriesM(filename=setFn)
self.assertEqual(testSet2.getSize(), 3)
for tsObj in testSet2:
self.assertEqual(tsObj.getSize(), 24)
testSet2.close()
return testSet
def _restimateCTF(self, ctfId):
""" Run Gctf with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(
pwutils.removeBaseExt(micFn) + '_EPA.log')
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = out
pwutils.cleanPath(psdFile)
try:
self.runJob(self._getProgram(), self._args % self._params)
except:
print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.cleanPattern(micFnMrc)
def detectCarbonCloseness(self, coordinates):
self.scoreCarbon = []
for tomoName in self.tomoNames:
idt = self.tomo_vesicles[tomoName]["volId"]
tomo = self.tomos[idt].clone()
projFile = self.projectTomo(tomo)
inputTomoPathMetadataFname = self._getTmpPath("inputTomo.xmd")
tomo_md = createMetaDataFromPattern(projFile)
tomo_md.write(inputTomoPathMetadataFname)
coordList = self.generateCoordList(tomo, coordinates)
self.writeTomoCoordinates(tomo, coordList,
self._getTomoPos(projFile))
args = '-i %s -c %s -o %s -b %d' \
% (inputTomoPathMetadataFname, self._getExtraPath('inputCoords'),
self._getExtraPath('outputCoords'), coordinates.getBoxSize())
self.runJob('xmipp_deep_micrograph_cleaner',
args,
env=Plugin.getTensorFlowEnviron())
baseName = pwutils.removeBaseExt(projFile)
outFile = self._getExtraPath('outputCoords', baseName + ".pos")
posMd = readPosCoordinates(outFile)
posMd.addLabel(md.MDL_ITEM_ID)
for objId in posMd:
if posMd.getValue(md.MDL_ENABLED, objId) == 0:
posMd.setValue(md.MDL_ENABLED, 1, objId)
coord = rowToCoordinate(rowFromMd(posMd, objId))
self.scoreCarbon.append([
posMd.getValue(md.MDL_ITEM_ID, objId),
coord._xmipp_goodRegionScore.get()
])
pwutils.cleanPath(projFile)
def extractMicrographStep(self, micKey, *args):
""" Step function that will be common for all extraction protocols.
It will take an id and will grab the micrograph from a micDict map.
The micrograph will be passed as input to the _extractMicrograph
function.
"""
# Retrieve the corresponding micrograph with this key and the
# associated list of coordinates
mic = self.micDict[micKey]
micDoneFn = self._getMicDone(mic)
micFn = mic.getFileName()
if self.isContinued() and os.path.exists(micDoneFn):
self.info("Skipping micrograph: %s, seems to be done" % micFn)
return
coordList = self.coordDict[mic.getObjId()]
self._convertCoordinates(mic, coordList)
# Clean old finished files
pwutils.cleanPath(micDoneFn)
self.info("Extracting micrograph: %s " % micFn)
self._extractMicrograph(mic, *args)
# Mark this mic as finished
open(micDoneFn, 'w').close()
def runTrainingStep(self):
# Create output folder
pwutils.cleanPath(self.getOutputDir())
pwutils.makePath(self.getOutputDir())
# Call cryoDRGN with the appropriate parameters.
self._runProgram('train_vae', self._getTrainingArgs())
def show(self, form):
prot = form.protocol
micSet = prot.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = prot.getProject()
micfn = micSet.getFileName()
# Prepare a temporary folder to convert some input files
# and put some of the intermediate result files
coordsDir = project.getTmpPath(micSet.getName())
pwutils.cleanPath(coordsDir)
pwutils.makePath(coordsDir)
prot.convertInputs(coordsDir)
pickerConfig = os.path.join(coordsDir, 'picker.conf')
f = open(pickerConfig, "w")
pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py')
pickCmd = prot._getPickArgs(threshold=False, workingDir=coordsDir)[0]
convertCmd = pw.join('apps', 'pw_convert.py')
args = {
"pickScript": pickScript,
"pickCmd": pickCmd,
"convertCmd": convertCmd,
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
'thresholdLow': prot.thresholdLow,
'thresholdHigh': prot.thresholdHigh,
"useGPU": prot.useGPU
}
f.write("""
parameters = thresholdLow,thresholdHigh
thresholdLow.value = %(thresholdLow)s
thresholdLow.label = Threshold Low
thresholdLow.help = Low value cut-off
thresholdHigh.value = %(thresholdHigh)s
thresholdHigh.label = Threshold High
thresholdHigh.help = High value cut-off
autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh)
convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, prot,
mode=CoordinatesObjectView.MODE_AUTOMATIC,
pickerProps=pickerConfig).show()
process.wait()
myprops = pwutils.readProperties(pickerConfig)
if myprops['applyChanges'] == 'true':
form.setVar('thresholdLow', myprops['thresholdLow.value'])
form.setVar('thresholdHigh', myprops['thresholdHigh.value'])
def classifyStep(self):
""" Run MSA-CL and MSA-SUM from IMAGIC. """
inputFile = self.inputMSA.get().getParticlesStack()
inputFileBase = pwutils.removeExt(inputFile)
inputFileImg = inputFileBase + '.img'
inputFileHed = inputFileBase + '.hed'
pwutils.createLink(inputFileImg, self._getTmpPath("particles.img"))
pwutils.createLink(inputFileHed, self._getTmpPath("particles.hed"))
inputFn = "tmp/particles"
if self.doDownweight.get():
downweight = 'YES'
else:
downweight = 'NO'
self._params.update({
'particles': inputFn,
'eigs_num': self.numberOfFactors.get(),
'cls_num': self.numberOfClasses.get(),
'perc_ign': self.percentIgnore.get(),
'downweight': downweight,
'perc_ign_bad': self.percentIgnoreBad.get()
})
classDir = self._getPath(self.CLASS_DIR)
pwutils.cleanPath(classDir)
pwutils.makePath(classDir)
self.runTemplate('msa/msa-cls.b', self._params)
def extractMicrographStep(self, micKey, *args):
""" Step function that will be common for all extraction protocols.
It will take an id and will grab the micrograph from a micDict map.
The micrograph will be passed as input to the _extractMicrograph
function.
"""
# Retrieve the corresponding micrograph with this key and the
# associated list of coordinates
mic = self.micDict[micKey]
micDoneFn = self._getMicDone(mic)
micFn = mic.getFileName()
if self.isContinued() and os.path.exists(micDoneFn):
self.info("Skipping micrograph: %s, seems to be done" % micFn)
return
coordList = self.coordDict[mic.getObjId()]
self._convertCoordinates(mic, coordList)
# Clean old finished files
pwutils.cleanPath(micDoneFn)
self.info("Extracting micrograph: %s " % micFn)
self._extractMicrograph(mic, *args)
# Mark this mic as finished
open(micDoneFn, 'w').close()
def nestedFlatDb(self):
fn = 'classes_flat.sqlite'
cleanPath(fn)
images = SetOfImages()
images.setSamplingRate(1.2)
classes2DSet = SetOfClasses2D(filename=fn)
classes2DSet.setImages(images)
for ref in range(1, 11):
print "class: ", ref
class2D = Class2D()
class2D.setObjId(ref)
print "append class to set, ref=", ref
classes2DSet.append(class2D)
avg = Particle()
avg.setLocation(ref, 'averages.stk')
print " populating class "
for i in range(1, 101):
img = Particle()
img.setSamplingRate(5.3)
class2D.append(img)
print " writing class "
class2D.write()
print " append avg"
averages.append(avg)
classes2DSet.write()
def _restimateCTF(self, ctfId):
""" Run ctffind3 with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = out
self._params['ctffindPSD'] = psdFile
pwutils.cleanPath(psdFile)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def close(self, e=None):
try:
print("Deleting temporary folder: ", customUserData)
pwutils.cleanPath(customUserData)
except Exception as ex:
print("Error: ", ex)
ProjectWindow.close(self, e)
def msaStep(self):
""" Run MSA on input particles. """
distances = ['EUCLIDIAN', 'CHISQUARE', 'MODULATION']
distance_name = distances[self.distanceType.get()]
self._params.update({
'msa_dir':
self.MSA_DIR,
'msa_distance':
distance_name,
'num_factors':
self.numberOfFactors.get(),
'num_iter':
self.numberOfIterations.get(),
'overcorrectionFactor':
self.overcorrectionFactor.get(),
'mpi_procs':
self.numberOfMpi.get()
})
msaDir = self._getPath(self.MSA_DIR)
if exists(msaDir):
pwutils.cleanPath(msaDir)
pwutils.makePath(msaDir)
self.runTemplate('msa/msa-run.b', self._params)
def test_convertMovie(self):
"""Check movie conversion"""
movFn = self.dsFormat.getFile('qbeta/qbeta.mrc') + ":mrcs"
ih = emlib.image.ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (4096, 4096, 1, 7)
EXPECTED_DT = emlib.DT_USHORT
self.assertEqual(ih.getDimensions(movFn), EXPECTED_SIZE)
self.assertEqual(ih.getDataType(movFn), EXPECTED_DT)
outFn = join('/tmp/qbeta_converted.mrcs')
ih.convertStack(movFn, outFn, 2, 6)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
self.assertEqual(ih.getDimensions(outFn), (4096, 4096, 1, 5))
self.assertEqual(ih.getDataType(outFn), EXPECTED_DT)
if pwutils.envVarOn(SCIPION_DEBUG_NOCLEAN):
print("Not cleaning output movie: ", outFn)
else:
pwutils.cleanPath(outFn)
def eliminationStep(self, fnInputMd):
self.inputImages = self.inputParticles.get()
partsFile = self.inputImages.getFileName()
self.partsSet = SetOfParticles(filename=partsFile)
self.partsSet.loadAllProperties()
self.streamClosed = self.partsSet.isStreamClosed()
if self.check == None:
writeSetOfParticles(self.partsSet,
fnInputMd,
alignType=em.ALIGN_NONE,
orderBy='creation')
else:
writeSetOfParticles(self.partsSet,
fnInputMd,
alignType=em.ALIGN_NONE,
orderBy='creation',
where='creation>"' + str(self.check) + '"')
for p in self.partsSet.iterItems(orderBy='creation', direction='DESC'):
self.check = p.getObjCreation()
break
self.partsSet.close()
self.lenPartsSet = len(self.partsSet)
print("os.path.exists(fnInputMd)): %s" % os.path.exists(fnInputMd))
args = "-i %s -o %s -e %s -t %f" % (
fnInputMd, self.fnOutputMd, self.fnElimMd, self.threshold.get())
if self.addFeatures:
args += " --addFeatures"
if self.useDenoising:
args += " --useDenoising -d %f" % self.denoising.get()
self.runJob("xmipp_image_eliminate_empty_particles", args)
cleanPath(fnInputMd)
def test_readDM4(self):
""" Check we can read dm4 files (using EMAN)
"""
micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4')
ih = emlib.image.ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (7676, 7420, 1, 1)
self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)
# We could even convert to an mrc file:
outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')
outFn = join('/tmp', outSuffix)
print("Converting: \n%s -> %s" % (micFn, outFn))
ih.convert(micFn, outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
# Check dimensions are still the same:
self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)
# Clean up tmp files
pwutils.cleanPath(outFn)
def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
self.finished = self.streamClosed and \
self.outputSize == self.inputSize
streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN
newData = os.path.exists(self.fnOutputMd)
lastToClose = self.finished and hasattr(self, 'outputParticles')
if newData or lastToClose:
outSet = self._loadOutputSet(SetOfParticles, 'outputParticles.sqlite')
if newData:
partsSet = self._createSetOfParticles()
readSetOfParticles(self.fnOutputMd, partsSet)
outSet.copyItems(partsSet)
for item in partsSet:
self._calculateSummaryValues(item)
self._store()
writeSetOfParticles(outSet.iterItems(orderBy='_xmipp_zScore'),
self._getPath("images.xmd"),
alignType=ALIGN_NONE)
cleanPath(self.fnOutputMd)
self._updateOutputSet('outputParticles', outSet, streamMode)
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(cons.STATUS_NEW)
def nestedFlatDb(self):
fn = 'classes_flat.sqlite'
cleanPath(fn)
images = SetOfImages()
images.setSamplingRate(1.2)
classes2DSet = SetOfClasses2D(filename=fn)
classes2DSet.setImages(images)
for ref in range(1, 11):
print "class: ", ref
class2D = Class2D()
class2D.setObjId(ref)
print "append class to set, ref=", ref
classes2DSet.append(class2D)
avg = Particle()
avg.setLocation(ref, 'averages.stk')
print " populating class "
for i in range(1, 101):
img = Particle()
img.setSamplingRate(5.3)
class2D.append(img)
print " writing class "
class2D.write()
print " append avg"
averages.append(avg)
classes2DSet.write()
def scipion_split_particle_stacks(inputStar, inputStack, output, filename_prefix, deleteStack):
""" Read a STAR file with particles and write as individual images.
If a stack of images is given, use these instead of the images from the STAR file.
Also write a new STAR file pointing to these images.
This function requires that the script is run within Scipion Python environment. """
import pyworkflow.utils as pwutils
from pyworkflow.em import ImageHandler
ih = ImageHandler()
md = MetaData(inputStar)
md.addLabels('rlnOriginalName')
# Initialize progress bar
progressbar = ProgressBar(width=60, total=len(md))
for i, particle in enumerate(md, start=1):
outputImageName = '%s/%s_%06d.mrc' % (output, filename_prefix, i)
if inputStack:
ih.convert((i, inputStack), outputImageName )
particle.rlnOriginalName = '%s/%06d@%s' %(output, i, inputStack)
else:
ih.convert(particle.rlnImageName, outputImageName)
particle.rlnOriginalName = particle.rlnImageName
particle.rlnImageName = outputImageName
progressbar.notify()
print("\n")
md.write("%s/%s.star" % (output, filename_prefix))
if inputStack and deleteStack:
pwutils.cleanPath(inputStack)
def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
self.finished = self.streamClosed and \
self.inputSize == self.outputSize
streamMode = emobj.Set.STREAM_CLOSED if self.finished else emobj.Set.STREAM_OPEN
# we will read all ready files
files = pwutils.glob(self.getTmpOutputPath('*'))
newData = len(files) > 0
lastToClose = self.finished and hasattr(self, 'outputCoordinates')
if newData or lastToClose:
outSet = self._loadOutputSet()
if newData:
for tmpFile in files:
tmpSet = emobj.SetOfCoordinates(filename=tmpFile)
tmpSet.loadAllProperties()
outSet.copyItems(tmpSet)
outSet.setBoxSize(tmpSet.getBoxSize())
tmpSet.close()
pwutils.cleanPath(tmpFile)
self._updateOutputSet('outputCoordinates',
outSet,
state=streamMode)
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(STATUS_NEW)
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
pickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(pickerProps, "w")
params = ['boxSize', 'lowerThreshold', 'higherThreshold', 'gaussWidth']
args = {
"params": ','.join(params),
"preprocess": "%s sxprocess.py" % pw.getScipionScript(),
"picker": "%s e2boxer.py" % pw.getScipionScript(),
"convert": pw.join('apps', 'pw_convert.py'),
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
"boxSize": autopickProt.boxSize,
"lowerThreshold": autopickProt.lowerThreshold,
"higherThreshold": autopickProt.higherThreshold,
"gaussWidth": autopickProt.gaussWidth,
"extraParams": autopickProt.extraParams
}
f.write("""
parameters = %(params)s
boxSize.value = %(boxSize)s
boxSize.label = Box Size
boxSize.help = some help
lowerThreshold.value = %(lowerThreshold)s
lowerThreshold.label = Lower Threshold
lowerThreshold.help = some help
higherThreshold.help = some help
higherThreshold.value = %(higherThreshold)s
higherThreshold.label = Higher Threshold
gaussWidth.help = some help
gaussWidth.value = %(gaussWidth)s
gaussWidth.label = Gauss Width
runDir = %(coordsDir)s
preprocessCommand = %(preprocess)s demoparms --makedb=thr_low=%%(lowerThreshold):thr_hi=%%(higherThreshold):boxsize=%%(boxSize):gauss_width=%%(gaussWidth):%(extraParams)s
autopickCommand = %(picker)s --gauss_autoboxer=demoparms --write_dbbox --boxsize=%%(boxSize) --norm=normalize.ramp.normvar %%(micrograph)
convertCommand = %(convert)s --coordinates --from eman2 --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project,
micfn,
coordsDir,
autopickProt,
pickerProps=pickerProps).show()
process.wait()
myprops = readProperties(pickerProps)
if myprops['applyChanges'] == 'true':
for param in params:
form.setVar(param, myprops[param + '.value'])
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
# Get current values of the properties
# micfn = os.path.join(coordsDir, 'micrographs.xmd')
# writeSetOfMicrographs(micSet, micfn)
dogpickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(dogpickerProps, "w")
args = {
"dogpicker":
os.path.join(os.environ['DOGPICKER_HOME'], "ApDogPicker.py"),
"convert":
pw.join('apps', 'pw_convert.py'),
'coordsDir':
coordsDir,
'micsSqlite':
micSet.getFileName(),
"diameter":
autopickProt.diameter,
"threshold":
autopickProt.threshold,
"apix":
micSet.getSamplingRate()
}
f.write("""
parameters = diameter,threshold
diameter.value = %(diameter)s
diameter.label = Diameter
diameter.help = some help
threshold.value = %(threshold)s
threshold.label = Threshold
threshold.help = some help
autopickCommand = %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameter) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt
convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project,
micfn,
coordsDir,
autopickProt,
pickerProps=dogpickerProps).show()
process.wait()
# Check if the wizard changes were accepted or just canceled
myprops = readProperties(dogpickerProps)
if myprops['applyChanges'] == 'true':
form.setVar('diameter', myprops['diameter.value'])
form.setVar('threshold', myprops['threshold.value'])
def show(self, form):
prot = form.protocol
micSet = prot.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = prot.getProject()
micfn = micSet.getFileName()
# Prepare a temporary folder to convert some input files
# and put some of the intermediate result files
coordsDir = project.getTmpPath(micSet.getName())
pwutils.cleanPath(coordsDir)
pwutils.makePath(coordsDir)
prot.convertInputs(coordsDir)
pickerConfig = os.path.join(coordsDir, 'picker.conf')
f = open(pickerConfig, "w")
pickScript = pw.join('em', 'packages', 'igbmc', 'run_gempicker.py')
pickCmd = prot.getArgs(threshold=False, workingDir=coordsDir)
convertCmd = pw.join('apps', 'pw_convert.py')
args = {
"pickScript": pickScript,
"pickCmd": pickCmd,
"convertCmd": convertCmd,
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
'thresholdLow': prot.thresholdLow,
'thresholdHigh': prot.thresholdHigh,
"useGPU": prot.useGPU
}
f.write("""
parameters = thresholdLow,thresholdHigh
thresholdLow.value = %(thresholdLow)s
thresholdLow.label = Threshold Low
thresholdLow.help = Low value cut-off
thresholdHigh.value = %(thresholdHigh)s
thresholdHigh.label = Threshold High
thresholdHigh.help = High value cut-off
autopickCommand = %(pickScript)s %%(micrograph) %(coordsDir)s %(useGPU)s %(pickCmd)s --thresh=%%(thresholdLow) --threshHigh=%%(thresholdHigh)
convertCommand = %(convertCmd)s --coordinates --from gempicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, prot,
pickerProps=pickerConfig).show()
process.wait()
myprops = pwutils.readProperties(pickerConfig)
if myprops['applyChanges'] == 'true':
form.setVar('thresholdLow', myprops['thresholdLow.value'])
form.setVar('thresholdHigh', myprops['thresholdHigh.value'])
def decompress(program, args, ext, nExt):
movieFolder = self._getTmpPath()
movieName = basename(movie.getFileName())
movieTmpLink = join(movieFolder, movieName)
pwutils.cleanPath(movieTmpLink)
pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
self.runJob(program, args % movieName, cwd=movieFolder)
dimMovie.setFileName(movieTmpLink.replace(ext, nExt))
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
micPath = micSet.getFileName()
convert.writeSetOfCoordinatesXmipp(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def decompress(program, args, ext, nExt):
movieFolder = self._getTmpPath()
movieName = basename(movie.getFileName())
movieTmpLink = join(movieFolder, movieName)
pwutils.cleanPath(movieTmpLink)
pwutils.createAbsLink(os.path.abspath(movieFn), movieTmpLink)
self.runJob(program, args % movieName, cwd=movieFolder)
dimMovie.setFileName(movieTmpLink.replace(ext, nExt))
def __readParticles(self, partsStar, outputSqlite=None, **kwargs):
outputSqlite = outputSqlite or self.getOutputPath('particles.sqlite')
print("<<< Reading star file: \n %s\n" % partsStar)
cleanPath(outputSqlite)
print(">>> Writing to particles db: \n %s\n" % outputSqlite)
partsSet = SetOfParticles(filename=outputSqlite)
convert.readSetOfParticles(partsStar, partsSet, **kwargs)
return partsSet
def writeVolumesSqlite(volsXml, volsSqlite, **kwargs):
""" Convert a volume list from PyTom xml format to Scipion sqlite file.
If the volSqlite exists, it will be deleted.
"""
pwutils.cleanPath(volsSqlite)
volSet = em.SetOfVolumes(filename=volsSqlite)
readSetOfVolumes(volsXml, volSet, **kwargs)
volSet.write()
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(
self._getPath(inputFit),
self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(
self._getPath(
'movie_particles_shiny_%sclass001_unfil.mrc' %
half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
half))
self._renameFiles('movie_particles_shiny_post*',
'movie_particles_')
self._renameFiles('movie_particles_shiny*',
'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(
mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def runCase(self, args, mpi=0, changeDir=False,
preruns=None, postruns=None, validate=None,
outputs=None, random=False):
# Retrieve the correct case number from the test name id
# We asumme here that 'test_caseXXX' should be in the name
caseId = unittest.TestCase.id(self)
if not 'test_case' in caseId:
raise Exception("'test_case' string should be in the test function name followed by a number")
_counter = int(caseId.split('test_case')[1])
self._testDir = self.dataset.getPath()
self.outputDir = os.path.join('tmpLink', '%s_%02d' % (self.program, _counter))
self.outputDirAbs = os.path.join(self._testDir, self.outputDir)
self.goldDir = os.path.join(self._testDir, 'gold', '%s_%02d' % (self.program, _counter))
# Clean and create the program output folder if not exists
pwutils.cleanPath(self.outputDirAbs)
pwutils.makePath(self.outputDirAbs)
# Change to tests root folder (self._testDir)
cwd = os.getcwd()
os.chdir(self._testDir)
if preruns:
self._runCommands(preruns, 'preruns')
if mpi:
cmd = "mpirun -np %d `which %s`" % (mpi, self.program)
else:
cmd = self.program
args = self._parseArgs(args)
if changeDir:
cmd = "cd %s ; %s %s > stdout.txt 2> stderr.txt" % (self.outputDir, cmd, args)
else:
cmd = "%s %s > %s/stdout.txt 2> %s/stderr.txt" % (cmd, args, self.outputDir, self.outputDir)
print " Command: "
print " ", pwutils.green(cmd)
#run the test itself
command = Command(cmd, env=self.env)
self._command = command
try:
command.run(timeout=self._timeout)
except KeyboardInterrupt:
command.terminate()
if postruns:
self._runCommands(postruns, 'postruns')
if outputs:
self._checkOutputs(outputs,random)
if validate:
validate()
os.chdir(cwd)
def exportCoordsStep(self, coordsId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
"""
pwutils.cleanPath(self._getExportPath())
pwutils.makePath(self._getExportPath())
convert.writeSetOfCoordinates(self._getExportPath(), self.getCoords(),
self._getMicPos)
def filterStep(self, args):
""" Apply the selected filter to particles.
Create the set of particles.
"""
particlesStk = self._getPath("particles.spi")
tmpStk = particlesStk.replace(".spi", "_tmp.spi")
self.runJob("bfilter", args + " %s %s" % (particlesStk, tmpStk))
pwutils.moveFile(tmpStk, particlesStk.replace(".spi", ".stk")) # just we prefer stk as stack of spider images
pwutils.cleanPath(particlesStk)
def filterStep(self, args):
""" Apply the selected filter to particles.
Create the set of particles.
"""
particlesStk = self._getPath('particles.spi')
tmpStk = particlesStk.replace('.spi', '_tmp.spi')
self.runJob('bfilter', args + ' %s %s' % (particlesStk, tmpStk))
pwutils.moveFile(tmpStk, particlesStk.replace('.spi', '.stk')) # just we prefer stk as stack of spider images
pwutils.cleanPath(particlesStk)
def close(self, e=None):
try:
print "Writing protocols to: ", jsonFn
proj.getRunsGraph() # Build project runs graph
proj.exportProtocols(proj.getRuns(), jsonFn)
print "Deleting temporary folder: ", customUserData
pwutils.cleanPath(customUserData)
except Exception, ex:
print "Error saving the workflow: ", ex
def __createSet(self, SetClass, template, suffix, **kwargs):
""" Create a set and set the filename using the suffix.
If the file exists, it will be delete. """
setFn = self._getPath(template % suffix)
# Close the connection to the database if
# it is open before deleting the file
pwutils.cleanPath(setFn)
setObj = SetClass(filename=setFn, **kwargs)
return setObj
def close(self, e=None):
try:
print "Writing protocols to: ", jsonFn
proj.getRunsGraph() # Build project runs graph
proj.exportProtocols(proj.getRuns(), jsonFn)
print "Deleting temporary folder: ", customUserData
pwutils.cleanPath(customUserData)
except Exception, ex:
print "Error saving the workflow: ", ex
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = micSet.getFileName()
xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = micSet.getFileName()
xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def deformationMatrix(self, volList):
cleanPath(self._getTmpPath("*.mrc"))
numVol = len(volList)
self.distanceMatrix = np.zeros((numVol, numVol))
for i in range(numVol):
for j in range(numVol):
if i != j:
path = self._getExtraPath('Pair_%d_%d_deformation.txt' %
(i, j))
self.distanceMatrix[i, j] = np.loadtxt(path)
def test_createEmptyFile(self):
x, y, z, n = 10, 10, 1, 100
img = Image()
for ext in ['stk', 'mrcs']:
imgPath = testFile('empty_100.%s' % ext)
createEmptyFile(imgPath, x, y, z, n)
img.read(imgPath, HEADER)
self.assertEqual(img.getDimensions(), (x, y, z, n))
pwutils.cleanPath(imgPath)
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = os.path.join(coordPath, 'micrographs.xmd')
xmipp3.writeSetOfMicrographs(micSet, micPath)
xmipp3.writeSetOfCoordinates(coordPath, coordSet)
return micPath, coordPath
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
pickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(pickerProps, "w")
params = ['boxSize', 'lowerThreshold', 'higherThreshold', 'gaussWidth']
args = {
"params": ','.join(params),
"preprocess" : os.path.join(os.environ['SCIPION_HOME'], "scipion sxprocess.py"),
"picker" : os.path.join(os.environ['SCIPION_HOME'], "scipion e2boxer.py"),
"convert" : os.path.join(os.environ['SCIPION_HOME'], os.path.join('pyworkflow','apps', 'pw_convert.py')),
'coordsDir':coordsDir,
'micsSqlite': micSet.getFileName(),
"boxSize": autopickProt.boxSize,
"lowerThreshold": autopickProt.lowerThreshold,
"higherThreshold": autopickProt.higherThreshold,
"gaussWidth": autopickProt.gaussWidth,
"extraParams":autopickProt.extraParams
}
f.write("""
parameters = %(params)s
boxSize.value = %(boxSize)s
boxSize.label = Box Size
boxSize.help = some help
lowerThreshold.value = %(lowerThreshold)s
lowerThreshold.label = Lower Threshold
lowerThreshold.help = some help
higherThreshold.help = some help
higherThreshold.value = %(higherThreshold)s
higherThreshold.label = Higher Threshold
gaussWidth.help = some help
gaussWidth.value = %(gaussWidth)s
gaussWidth.label = Gauss Width
runDir = %(coordsDir)s
preprocessCommand = %(preprocess)s demoparms --makedb=thr_low=%%(lowerThreshold):thr_hi=%%(higherThreshold):boxsize=%%(boxSize):gauss_width=%%(gaussWidth):%(extraParams)s
autopickCommand = %(picker)s --gauss_autoboxer=demoparms --write_dbbox --boxsize=%%(boxSize) --norm=normalize.ramp.normvar %%(micrograph)
convertCommand = %(convert)s --coordinates --from eman2 --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt, pickerProps=pickerProps).show()
process.wait()
myprops = readProperties(pickerProps)
for param in params:
form.setVar(param, myprops[param + '.value'])
def _fixMovie(self, movie):
if self.doSaveMovie and self.useMotioncor2 and self._isOutStackSupport():
outputMicFn = self._getExtraPath(self._getOutputMicName(movie))
outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
movieFn = outputMicFn.replace('_aligned_mic.mrc',
'_aligned_mic_Stk.mrc')
pwutils.moveFile(movieFn, outputMovieFn)
if self.useMotioncor2 and not self.doSaveUnweightedMic:
fnToDelete = self._getExtraPath(self._getOutputMicName(movie))
pwutils.cleanPath(fnToDelete)
def show(self, form):
autopickProt = form.protocol
micSet = autopickProt.getInputMicrographs()
if not micSet:
print 'must specify input micrographs'
return
project = autopickProt.getProject()
micfn = micSet.getFileName()
coordsDir = project.getTmpPath(micSet.getName())
cleanPath(coordsDir)
makePath(coordsDir)
# Get current values of the properties
# micfn = os.path.join(coordsDir, 'micrographs.xmd')
# writeSetOfMicrographs(micSet, micfn)
dogpickerProps = os.path.join(coordsDir, 'picker.conf')
f = open(dogpickerProps, "w")
args = {
"dogpicker" : os.path.join(os.environ['DOGPICKER_HOME'], "ApDogPicker.py"),
"convert" : pw.join('apps', 'pw_convert.py'),
'coordsDir': coordsDir,
'micsSqlite': micSet.getFileName(),
"diameter": autopickProt.diameter,
"threshold": autopickProt.threshold,
"apix": micSet.getSamplingRate()
}
f.write("""
parameters = diameter,threshold
diameter.value = %(diameter)s
diameter.label = Diameter
diameter.help = some help
threshold.value = %(threshold)s
threshold.label = Threshold
threshold.help = some help
autopickCommand = %(dogpicker)s --thresh=%%(threshold) --diam=%%(diameter) --apix=%(apix)s --image=%%(micrograph) --outfile=%(coordsDir)s/%%(micrographName).txt
convertCommand = %(convert)s --coordinates --from dogpicker --to xmipp --input %(micsSqlite)s --output %(coordsDir)s
""" % args)
f.close()
process = CoordinatesObjectView(project, micfn, coordsDir, autopickProt,
mode=CoordinatesObjectView.MODE_AUTOMATIC,
pickerProps=dogpickerProps).show()
process.wait()
# Check if the wizard changes were accepted or just canceled
myprops = readProperties(dogpickerProps)
if myprops['applyChanges'] == 'true':
form.setVar('diameter', myprops['diameter.value'])
form.setVar('threshold', myprops['threshold.value'])
def writeZeroShifts(self, movie):
# TODO: find another way to do this
shiftsMd = self._getTmpPath('zero_shifts.xmd')
pwutils.cleanPath(shiftsMd)
xshifts = [0] * movie.getNumberOfFrames()
yshifts = xshifts
alignment = MovieAlignment(first=1, last=movie.getNumberOfFrames(),
xshifts=xshifts, yshifts=yshifts)
roiList = [0, 0, 0, 0]
alignment.setRoi(roiList)
movie.setAlignment(alignment)
writeShiftsMovieAlignment(movie, shiftsMd,
1, movie.getNumberOfFrames())
return shiftsMd
def main():
parser = argparse.ArgumentParser(prog='Scipion Convert')
parser.add_argument('--coordinates', help='Convert coordinates', action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder')
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
elif fromType == 'dogpicker':
if toType == 'xmipp':
from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
elif fromType == 'relion':
if toType == 'xmipp':
def readSetOfCoordinates(outputDir, micSet, coordSet):
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
inputCoords = args.extra
starFiles = [os.path.join(inputCoords,
pwutils.removeBaseExt(mic.getFileName())
+ '_autopick.star') for mic in micSet]
readSetOfCoordinates(coordSet, starFiles)
elif fromType == 'gautomatch':
if toType == 'xmipp':
from pyworkflow.em.packages.gautomatch.convert import readSetOfCoordinates
elif fromType == 'gempicker':
if toType == 'xmipp':
from pyworkflow.em.packages.igbmc.convert import readSetOfCoordinates
else:
raise Exception('Unknown coordinates type: %s' % fromType)
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinatesWithState
writeSetOfCoordinatesWithState(outputDir, coordSet, state='Automatic')
def main():
parser = argparse.ArgumentParser(prog='Scipion Convert')
parser.add_argument('--coordinates', help='Convert coordinates', action="store_true")
parser.add_argument('--fromType', help='Convert from input type')
parser.add_argument('--toType', help='Convert to output type')
parser.add_argument('--input', help='Input file or folder')
parser.add_argument('--output', help='Output file or folder')
parser.add_argument('--extra', help='To add extra parameters')
args = parser.parse_args()
fromType = args.fromType
toType = args.toType
input = args.input
output = args.output
if args.coordinates:
#print 'converting coordinates ...'
micSet = loadSetFromDb(input)
outputDir = output
coordsfn = os.path.join(outputDir, 'coordinates.sqlite')
cleanPath(coordsfn)
coordSet = SetOfCoordinates(filename=coordsfn)
coordSet.setMicrographs(micSet)
if fromType == 'eman2':
if toType == 'xmipp':
#print 'from eman2 to xmipp...'
from pyworkflow.em.packages.eman2.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'dogpicker':
if toType == 'xmipp':
#print 'from dogpicker to xmipp...'
from pyworkflow.em.packages.appion.convert import readSetOfCoordinates
readSetOfCoordinates(outputDir, micSet, coordSet)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
if fromType == 'relion':
if toType == 'xmipp':
#print 'from relion to xmipp...'
inputCoords = args.extra
starFiles = [os.path.join(inputCoords, pwutils.removeBaseExt(mic.getFileName()) + '_autopick.star')
for mic in micSet]
from pyworkflow.em.packages.relion.convert import readSetOfCoordinates
readSetOfCoordinates(coordSet, starFiles)
from pyworkflow.em.packages.xmipp3.convert import writeSetOfCoordinates
writeSetOfCoordinates(outputDir, coordSet, ismanual=False)
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _getClassesSqlite(self, blockName, fn):
""" Read the classes from Xmipp metadata and write as sqlite file. """
fnSqlite = fn + "_" + blockName + ".sqlite"
# Generate the sqlite file from classes xmd if either
# 1) Sqlite files has not been generated or
# 2) Xmd file is newer than sqlite (using modification time)
if os.path.exists(fnSqlite):
if os.path.getmtime(fn) > os.path.getmtime(fnSqlite) - 10:
pwutils.cleanPath(fnSqlite) # Clean to load from scratch
if not os.path.exists(fnSqlite):
classesSet = SetOfClasses2D(filename=fnSqlite)
classesSet.setImages(self._getInputParticles())
readSetOfClasses(classesSet, fn, blockName, preprocessClass=self._setClassOrder)
classesSet.write()
return fnSqlite
def runGempicker(micName, workingDir, useGPU, args, log=None):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
refDir = join(workingDir, 'templates')
maskSchDir = join(workingDir, 'maskSch')
args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir,
refDir, maskSchDir)
# Run Gempicker:
for mode in [0, 1]:
pwutils.runJob(log, getProgram(useGPU), args + ' --mode=%d' % mode,
env=getEnviron())
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def main(self):
self.define_parser()
args = self.parser.parse_args()
# Validate input arguments and required software (Bsoft)
# self.validate(args)
# Load subparticle vectors either from Chimera CMM file or from
# command line (command and semi-colon separated)
# Distances can also be specified to modify vector lengths
subparticle_vector_list = localrec.load_vectors(args.cmm, args.vector,
args.length, args.angpix)
size = args.particle_size
half = size / 2
v = subparticle_vector_list[0]
d = v.distance()
maskFn = 'mask.mrc'
v.scale(d)
phantomFn = 'phantom.descr'
phantomFile = open(phantomFn, 'w')
phantomFile.write("%d %d %d 1 1\n" % (size, size, size))
sym_matrices = localrec.matrix_from_symmetry(args.sym)
for m in sym_matrices:
vm = localrec.matrix_product(m, v)
sx, sy, sz = vm.data()
phantomFile.write("sph + -1 %d %d %d %d\n" % (sx, sy, sz, args.radius))
phantomFile.close()
runProgram('xmipp_phantom_create', '%s -o %s' % (phantomFn, maskFn))
pwutils.cleanPath(phantomFn)
runProgram('xmipp_transform_filter',
'%s --fourier real_gaussian %d ' % (maskFn, args.edge))
print "All done!"
print " "
print "Apply the mask mask.mrc on your particle."
print "The masked particle can be used for partial signal subtraction for localized reconstruction."
print " "
def msaStep(self):
""" Run MSA on input particles. """
distances = ['EUCLIDIAN', 'CHISQUARE', 'MODULATION']
distance_name = distances[self.distanceType.get()]
self._params.update({'msa_dir': self.MSA_DIR,
'msa_distance': distance_name,
'num_factors': self.numberOfFactors.get(),
'num_iter': self.numberOfIterations.get(),
'overcorrectionFactor': self.overcorrectionFactor.get(),
'mpi_procs': self.numberOfMpi.get()
})
msaDir = self._getPath(self.MSA_DIR)
if exists(msaDir):
pwutils.cleanPath(msaDir)
pwutils.makePath(msaDir)
self.runTemplate('msa/msa-run.b', self._params)
def exportParticlesStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
"""
imgSet = self.inputParticles.get()
self._stackType = self.stackType.get()
self._ih = em.ImageHandler()
self._stackDict = {}
particlesPath = self._getPath('Particles')
pwutils.cleanPath(particlesPath)
pwutils.makePath(particlesPath)
alignType = imgSet.getAlignment() if self.useAlignment else em.ALIGN_NONE
# Create links to binary files and write the relion .star file
writeSetOfParticles(imgSet, self._getPath("particles.star"),
outputDir=self._getExtraPath(),
alignType=alignType,
postprocessImageRow=self._postprocessImageRow,
fillMagnification=True)
pwutils.prettyDict(self._stackDict)
def pickMicrographListStep(self, micNameList, *args):
micList = []
for micName in micNameList:
mic = self.micDict[micName]
micDoneFn = self._getMicDone(mic)
micFn = mic.getFileName()
if self.isContinued() and os.path.exists(micDoneFn):
self.info("Skipping micrograph: %s, seems to be done" % micFn)
else:
# Clean old finished files
pwutils.cleanPath(micDoneFn)
self.info("Picking micrograph: %s " % micFn)
micList.append(mic)
self._pickMicrographList(micList, *args)
for mic in micList:
# Mark this mic as finished
open(self._getMicDone(mic), 'w').close()
def checkMovie():
for k, v in frameDict.iteritems():
moviePath = os.path.dirname(k)
movieFn = join(moviePath + "/", self._getUniqueFileName(k) +
suffix)
if self.writeMoviesInProject:
movieFn = self._getExtraPath(os.path.basename(movieFn))
if (movieFn not in self.importedFiles and
movieFn not in self.createdStacks and
len(v) == self.numberOfIndividualFrames):
movieOut = movieFn
if movieOut.endswith("mrc"):
movieOut += ":mrcs"
# By default we will write the movie stacks
# unless we are in continue mode and the file exists
writeMovie = True
if (self.isContinued() and os.path.exists(movieFn)):
self.info("Skipping movie stack: %s, seems to be done"
% movieFn)
writeMovie = False
if writeMovie:
self.info("Writing movie stack: %s" % movieFn)
# Remove the output file if exists
pwutils.cleanPath(movieFn)
for i, frame in enumerate(sorted(v, key=lambda x: x[0])):
frameFn = frame[1] # Frame name stored previously
ih.convert(frameFn, (i+1, movieOut))
if self.deleteFrames:
pwutils.cleanPath(frameFn)
# Now return the newly created movie file as imported file
self.createdStacks.add(movieFn)
return
