def getImportClass(self):
""" Return the class in charge of importing the files. """
filesPath = self.filesPath.get()
importFrom = self.importFrom.get()
if importFrom == self.IMPORT_FROM_AUTO:
importFrom = self.getFormat()
if importFrom == self.IMPORT_FROM_XMIPP3:
XmippImport = Domain.importFromPlugin('xmipp3.convert', 'XmippImport',
doRaise=True)
return XmippImport(self, filesPath)
elif importFrom == self.IMPORT_FROM_GRIGORIEFF:
GrigorieffLabImportCTF = Domain.importFromPlugin('cistem.convert',
'GrigorieffLabImportCTF',
doRaise=True)
return GrigorieffLabImportCTF(self)
elif importFrom == self.IMPORT_FROM_GCTF:
GctfImportCTF = Domain.importFromPlugin('gctf.convert',
'GctfImportCTF', doRaise=True)
return GctfImportCTF(self)
elif importFrom == self.IMPORT_FROM_EMAN2:
EmanImport = Domain.importFromPlugin('eman2.convert', 'EmanImport',
doRaise=True)
return EmanImport(self, None)
elif importFrom == self.IMPORT_FROM_SCIPION:
from .dataimport import ScipionImport
return ScipionImport(self, self.filesPath.get('').strip())
else:
return None
def getImportClass(self):
""" Return the class in charge of importing the files. """
filesPath = self.filesPath.get()
importFrom = self.getImportFrom()
if importFrom == self.IMPORT_FROM_XMIPP:
XmippImport = Domain.importFromPlugin('xmipp3.convert', 'XmippImport',
'Xmipp is needed to import .xmd files',
doRaise=True)
return XmippImport(self, filesPath)
elif importFrom == self.IMPORT_FROM_RELION:
RelionImport = Domain.importFromPlugin('relion.convert', 'RelionImport',
errorMsg='Relion is needed to import .star files',
doRaise=True)
return RelionImport(self, filesPath)
elif importFrom == self.IMPORT_FROM_EMAN:
EmanImport = Domain.importFromPlugin('eman2.convert', 'EmanImport',
errorMsg='Eman is needed to import .json or '
'.box files',
doRaise=True)
return EmanImport(self, None)
elif importFrom == self.IMPORT_FROM_DOGPICKER:
DogpickerImport = Domain.importFromPlugin('appion.convert', 'DogpickerImport',
errorMsg='appion plugin is needed to import '
'dogpicker files',
doRaise=True)
return DogpickerImport(self)
else:
self.importFilePath = ''
return None
def runProgram(program, params):
""" Runs a em program setting its environment matching a prefix"""
env = None
# Allow passing absolute paths
programName = basename(program)
from pwem import Domain
# Avoid detecting xmipp installation script to be run as xmipp, since
# it will run devel installation with production xmippEnv.json environment:
# Example: want to compile devel with java 8, but production was compiled with java 11.
# So java 11 makes it into the PATH taking priority
if programName.startswith('xmipp_'):
print("Xmipp command detected")
xmipp3 = Domain.getPlugin('xmipp3').Plugin
env = xmipp3.getEnviron()
if programName.startswith('relion'):
print("relion command detected")
relion = Domain.getPlugin("relion").Plugin
env = relion.getEnviron()
elif (programName.startswith('e2') or programName.startswith('sx')):
print("eman/sparx command detected")
eman2 = Domain.importFromPlugin('eman2', 'Plugin')
env = eman2.getEnviron()
elif programName.startswith('b'):
print("Bsoft command detected")
bsoft = Domain.importFromPlugin('bsoft', 'Plugin')
env = bsoft.getEnviron()
pwutils.runJob(None, program, params, env=env)
def test_pattern(self):
protocols = [] # append here all protocols to make final checks
# ----------- IMPORT MOVIES -------------------
protImport = self.newProtocol(
emprot.ProtImportMovies,
objLabel='import movies',
importFrom=emprot.ProtImportMovies.IMPORT_FROM_FILES,
filesPath=os.path.abspath(self.proj.getTmpPath()),
filesPattern="movie*%s" % self.ext,
amplitudConstrast=0.1,
sphericalAberration=2.,
voltage=300,
samplingRate=3.54,
dataStreaming=True,
timeout=TIMEOUT)
self.proj.launchProtocol(protImport, wait=False)
self._waitOutput(protImport, 'outputMovies')
protocols.append(protImport)
# ----------- ALIGNMENT --------------------------
XmippProtMovieCorr = Domain.importFromPlugin('xmipp3.protocols',
'XmippProtMovieCorr',
doRaise=True)
protOF = self.newProtocol(XmippProtMovieCorr,
objLabel='Movie alignment',
doSaveMovie=False,
doComputePSD=False,
alignFrame0=3,
alignFrameN=10,
sumFrame0=3,
sumFrameN=10,
doApplyDoseFilter=False)
protOF.inputMovies.set(protImport)
protOF.inputMovies.setExtended('outputMovies')
self.proj.launchProtocol(protOF, wait=False)
self._waitOutput(protOF, 'outputMicrographs')
protocols.append(protOF)
# --------- CTF ESTIMATION ---------------------------
ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
'CistemProtCTFFind',
doRaise=True)
protCTF = self.newProtocol(ProtCTFFind, objLabel='ctffind4')
protCTF.inputMicrographs.set(protOF)
protCTF.inputMicrographs.setExtended('outputMicrographs')
self.proj.scheduleProtocol(protCTF)
self._waitOutput(protCTF, 'outputCTF')
protocols.append(protCTF)
# --------- SUMMARY MONITOR --------------------------
protMonitor = self.newProtocol(ProtMonitorSummary, objLabel='summary')
protMonitor.inputProtocols.append(protImport)
protMonitor.inputProtocols.append(protOF)
protMonitor.inputProtocols.append(protCTF)
self.proj.launchProtocol(protMonitor, wait=False)
protocols.append(protMonitor)
# ---------- STREAMING CHECKS ------------------------
self.finalChecks(protImport.outputMovies, protocols)
def test_groel(self):
""" Run an Import particles protocol. """
cpus = os.environ.get('SCIPION_TEST_CPU', 4)
# 1. Run import of averages
groelAvg = self.ds.getFile('groel')
sym = 'd7'
protImport = self.newProtocol(ProtImportAverages,
objLabel='import averages (groel)',
filesPath=groelAvg,
samplingRate=1)
self.launchProtocol(protImport)
# 2. Run initial models
# 2a. Ransac
xmipp3 = Domain.importFromPlugin('xmipp3.protocols', doRaise=True)
protRansac = self.newProtocol(xmipp3.XmippProtRansac,
objLabel='xmipp - ransac',
symmetryGroup=sym,
numberOfMpi=1,
numberOfThreads=cpus
)
protRansac.inputSet.set(protImport.outputAverages)
self.launchProtocol(protRansac)
# 2b. Eman
eman2 = Domain.importFromPlugin('eman2.protocols', doRaise=True)
protEmanInitVol = self.newProtocol(eman2.EmanProtInitModel,
objLabel='eman - initial vol',
symmetry=sym,
numberOfThreads=cpus)
protEmanInitVol.inputSet.set(protImport.outputAverages)
self.launchProtocol(protEmanInitVol)
# 3. Significant
protSignificant = self.newProtocol(xmipp3.XmippProtReconstructSignificant,
objLabel='xmipp - significant',
symmetryGroup=sym,
numberOfMpi=cpus,
numberOfThreads=1,
iter=15,
alpha0=95
)
protSignificant.inputSet.set(protImport.outputAverages)
self.launchProtocol(protSignificant)
# 4. Align all volumes
protAlign = self.newProtocol(xmipp3.XmippProtAlignVolumeForWeb,
objLabel='xmipp - align volumes',
numberOfMpi=1,
numberOfThreads=cpus
)
protAlign.inputReference.set(protSignificant.outputVolume)
protAlign.inputVolumes.append(protRansac.outputVolumes)
protAlign.inputVolumes.append(protEmanInitVol.outputVolumes)
protAlign.inputVolumes.append(protSignificant.outputVolume)
self.launchProtocol(protAlign)
def getImportClass(self):
""" Return the class in charge of importing the files. """
if self.importFrom == self.IMPORT_FROM_EMX:
EmxImport = Domain.importFromPlugin(
'emxlib.convert',
'EmxImport',
errorMsg='Emx is needed to import .emx files',
doRaise=True)
self.importFilePath = abspath(self.emxFile.get('').strip())
return EmxImport(self, self.importFilePath,
self.alignTypeList[self.alignType.get()])
elif self.importFrom == self.IMPORT_FROM_XMIPP3:
XmippImport = Domain.importFromPlugin(
'xmipp3.convert',
'XmippImport',
'Xmipp is needed to import .xmd files',
doRaise=True)
self.importFilePath = self.mdFile.get('').strip()
return XmippImport(self, self.mdFile.get())
elif self.importFrom == self.IMPORT_FROM_RELION:
RelionImport = Domain.importFromPlugin(
'relion.convert',
'RelionImport',
errorMsg='Relion is needed to import .star files',
doRaise=True)
self.importFilePath = self.starFile.get('').strip()
return RelionImport(self, self.starFile.get())
elif self.importFrom == self.IMPORT_FROM_SCIPION:
from .dataimport import ScipionImport
self.importFilePath = self.sqliteFile.get('').strip()
return ScipionImport(self, self.importFilePath)
elif self.importFrom == self.IMPORT_FROM_FREALIGN:
self.importFilePath = self.parFile.get('').strip()
GrigorieffLabImportParticles = Domain.importFromPlugin(
'cistem.convert',
'GrigorieffLabImportParticles',
errorMsg='Cistem is needed to import .stk files',
doRaise=True)
return GrigorieffLabImportParticles(self, self.parFile.get(),
self.stackFile.get())
elif self.importFrom == self.IMPORT_FROM_EMAN:
self.importFilePath = self.lstFile.get('').strip()
EmanImport = Domain.importFromPlugin('eman2.convert',
'EmanImport',
doRaise=True)
return EmanImport(self, self.lstFile.get())
else:
self.importFilePath = ''
return None
def getImportClass(self):
""" Return the class in charge of importing the files. """
importFrom = self.importFrom.get()
if importFrom == self.IMPORT_FROM_XMIPP:
XmippImport = Domain.importFromPlugin('xmipp3.convert', 'XmippImport',
'Xmipp is needed to import .xmd files',
doRaise=True)
return XmippImport(self, None)
else: # import from EMAN
EmanImport = Domain.importFromPlugin('eman2.convert', 'EmanImport',
errorMsg='Eman is needed to import .json or '
'.box files',
doRaise=True)
return EmanImport(self, None)
def homoGoldStandardAlign(self, relion1, relion2):
"""test without Goldstandard with alignment"""
XmippProtVolumeHomogenizer = Domain.importFromPlugin(
'xmipp3.protocols', 'XmippProtEnrich', doRaise=True)
protVolumeHomogenizer = self.newProtocol(
XmippProtVolumeHomogenizer,
objLabel='volume homogenizer4',
doGoldStandard=True)
protVolumeHomogenizer.referenceVolume1.set(relion1.outputVolume)
protVolumeHomogenizer.inputVolume1.set(relion2.outputVolume)
protVolumeHomogenizer.referenceVolume2.set(relion1.outputVolume)
protVolumeHomogenizer.inputVolume2.set(relion2.outputVolume)
protVolumeHomogenizer.inputParticles.set(relion2.outputParticles)
protVolumeHomogenizer.doAlignment.set(True)
self.launchProtocol(protVolumeHomogenizer)
self.assertIsNotNone(
protVolumeHomogenizer.outputParticles01.getFileName(),
"There was a problem with the homoGoldStandardAlign")
self.assertIsNotNone(
protVolumeHomogenizer.outputParticles02.getFileName(),
"There was a problem with the homoGoldStandardAlign")
return protVolumeHomogenizer
def _runSignificant(self, inputSet, args):
myargs = dict(args)
xmipp3 = Domain.importFromPlugin('xmipp3.protocols', doRaise=True)
prot1 = self.newProtocol(xmipp3.XmippProtReconstructSignificant,
objLabel='significant d7',
**myargs
)
# prot1.inputClasses.set(inputSet)
self.launchProtocol(prot1)
output = prot1.outputVolume
myargs['thereisRefVolume'] = True
prot2 = self.newProtocol(xmipp3.XmippProtReconstructSignificant,
objLabel='significant d7 (with ref)',
**myargs
)
# prot2.inputClasses.set(inputSet)
prot2.refVolume.set(output)
self.launchProtocol(prot2)
# Run significant with one volume and new target resolution
myargs['useMaxRes'] = True
prot3 = self.newProtocol(xmipp3.XmippProtReconstructSignificant,
objLabel='significant d7 (ref and maxRes)',
**myargs
)
prot3.refVolume.set(output)
self.launchProtocol(prot3)
def __runEman2Program(self, program, args):
""" Internal workaround to launch an EMAN2 program. """
from pwem import Domain
eman2 = Domain.importFromPlugin('eman2')
from pyworkflow.utils.process import runJob
runJob(self._log, eman2.Plugin.getProgram(program), args,
env=eman2.Plugin.getEnviron())
def _visualize(self, obj, **args):
# The input pdb is a parameter from the protocol
# and from the parent protocol.
inputAtomStruct = self.protocol.pdbFileToBeRefined.get()
# To show pdbs only
dim = 150.
sampling = 1.
bildFileName = os.path.abspath(self.protocol._getTmpPath(
"axis_output.bild"))
Chimera.createCoordinateAxisFile(dim,
bildFileName=bildFileName,
sampling=sampling)
fnCmd = self.protocol._getTmpPath("chimera_output.cxc")
f = open(fnCmd, 'w')
f.write("open %s\n" % bildFileName)
f.write("cofr 0,0,0\n") # set center of coordinates
f.write("open %s\n"
% os.path.abspath(inputAtomStruct.getFileName()))
f.write("style stick\n")
if self.protocol.hasAttribute('rotatedAtomStruct'):
outputAtomStruct = self.protocol.rotatedAtomStruct.getFileName()
f.write("open %s\n" % os.path.abspath(outputAtomStruct))
f.close()
# run in the background
chimeraPlugin = Domain.importFromPlugin('chimera', 'Plugin', doRaise=True)
chimeraPlugin.runChimeraProgram(chimeraPlugin.getProgram(), fnCmd + "&")
return []
def _convertGain(self):
""" We need to transform gain file for a possible polishing job. """
rotation = self.gainRot.get()
flip = self.gainFlip.get()
gainFn = self.gainFile.get()
if rotation or flip:
args = "%s %s " % (gainFn, self._getPath(os.path.basename(gainFn)))
if flip:
# flip axis Y - left to right
args += "--process xform.flip:axis=%s " % ("y" if flip == 2 else "x")
if rotation:
args += "--rotate %d " % (rotation * 90)
from pwem import Domain
eman2 = Domain.importFromPlugin('eman2')
pwutils.runJob(self._log, eman2.Plugin.getProgram('e2proc2d.py'), args,
env=eman2.Plugin.getEnviron())
return self._getPath(os.path.basename(gainFn))
else:
outputGain = self._getPath(os.path.basename(gainFn))
pwutils.createAbsLink(gainFn, outputGain)
return outputGain
def _runRelionClassify2D(self, previousRun, label='', threads=1, MPI=1,
doGpu=False, GPUs='', useQueue=False, steps=False):
""" :param previousRun: The outputParticles of that will be the input
:param label: For naming purposes
:param threads: How many threads to use
:param MPI: How many MPIs to use
:param doGpu: Use GPU or not
:param GPUs: Which GPUs to use (see Relion gpusToUse form param)
:param useQueue: Use the queue system or not
:return: the launched protocol
"""
relionProtocols = Domain.importFromPlugin('relion.protocols',
doRaise=True)
prot2D = self.newProtocol(relionProtocols.ProtRelionClassify2D,
doCTF=False, maskDiameterA=340,
numberOfMpi=MPI, numberOfThreads=threads)
prot2D.numberOfClasses.set(4)
prot2D.numberOfIterations.set(3)
prot2D.inputParticles.set(previousRun.outputParticles)
prot2D.setObjLabel(label)
if useQueue:
prot2D._useQueue.set(True)
if steps:
QUEUE_PARAMS[1]['QUEUE_FOR_JOBS'] = 'Y'
prot2D._queueParams.set(json.dumps(QUEUE_PARAMS))
prot2D.doGpu.set(doGpu)
if doGpu:
prot2D.gpusToUse.set(GPUs)
self.launchProtocol(prot2D)
return prot2D
def registerCoords(self, coordsDir):
""" This method is usually inherited by all Pickers
and it is used from the Java picking GUI to register
a new SetOfCoordinates when the user click on +Particles button.
"""
from pwem import Domain
suffix = self.__getOutputSuffix()
outputName = self.OUTPUT_PREFIX + suffix
readSetOfCoordinates = Domain.importFromPlugin('xmipp3.convert',
'readSetOfCoordinates')
inputset = self.getInputMicrographs()
# micrographs are the input set if protocol is not finished
outputset = self._createSetOfCoordinates(inputset, suffix=suffix)
readSetOfCoordinates(coordsDir, outputset.getMicrographs(), outputset)
summary = self.getSummary(outputset)
outputset.setObjComment(summary)
outputs = {outputName: outputset}
self._defineOutputs(**outputs)
# Using a pointer to define the relations is more robust to scheduling
# and id changes between the protocol run.db and the main project
# database. The pointer defined below points to the outputset object
self._defineSourceRelation(
self.getInputMicrographsPointer(),
pwobj.Pointer(value=self, extended=outputName))
self._store()
def testStepsNoGPU(self):
xmipp3Protocols = Domain.importFromPlugin('xmipp3.protocols',
doRaise=True)
protXmippPreproc = self.newProtocol(xmipp3Protocols.XmippProtPreprocessParticles,
doNormalize=True, doRemoveDust=True)
protXmippPreproc.inputParticles.set(self.protImport.outputParticles)
protXmippPreproc.setObjLabel("Xmipp preprocess steps")
protXmippPreproc._useQueue.set(True)
QUEUE_PARAMS[1]['QUEUE_FOR_JOBS'] = 'Y'
protXmippPreproc._queueParams.set(json.dumps(QUEUE_PARAMS))
# Launch protocol but wait until it finishes
self.launchProtocol(protXmippPreproc, wait=True)
# Check that job files have been created
jobFilesPath = join(pwutils.getParentFolder(protXmippPreproc.getLogPaths()[0]),
str(protXmippPreproc.getObjId()))
self.assertTrue(pwutils.exists(jobFilesPath + "-0-1.out") and
pwutils.exists(jobFilesPath + "-0-1.err") and
pwutils.exists(jobFilesPath + "-0-1.job") and
pwutils.exists(jobFilesPath + "-0-2.out") and
pwutils.exists(jobFilesPath + "-0-2.err") and
pwutils.exists(jobFilesPath + "-0-2.job"),
"Job queue files not found on log folder, job did "
"not make it to the queue.")
# Check that results have been produced
self.assertIsNotNone(protXmippPreproc.outputParticles,
"There was a problem with Xmipp preprocess particles.")
def convert(self, inputObj, outputObj, dataType=None, transform=None):
""" Convert from one image to another.
inputObj and outputObj can be: tuple, string, or Image subclass
(see self._convertToLocation)
transform: if not None, apply this transformation
"""
inputLoc = self._convertToLocation(inputObj)
outputLoc = self._convertToLocation(outputObj)
if outputLoc[1].lower().endswith('.img'):
# FIXME Since now we can not read dm4 format in Scipion natively
# we are opening an Eman2 process to read the dm4 file
from pwem import Domain
convertImage = Domain.importFromPlugin('eman2.convert',
'convertImage',
doRaise=True)
convertImage(inputLoc, outputLoc)
else:
# Read from input
self._img.read(inputLoc)
if dataType is not None:
self._img.convert2DataType(dataType)
if transform is not None:
self._img.applyTransforMatScipion(transform.getMatrixAsList())
# Write to output
self._img.write(outputLoc)
def readCoordinates(self):
micTiltPairs = self.getInputMicrographs()
ci = self.getImportClass()
uSet = micTiltPairs.getUntilted()
tSet = micTiltPairs.getTilted()
# Create Untilted and Tilted SetOfCoordinates
uCoordSet = self._createSetOfCoordinates(uSet, suffix='Untilted')
tCoordSet = self._createSetOfCoordinates(tSet, suffix='Tilted')
anglesSet = self._createSetOfAngles()
def _importCoords(uCoordSet, tCoordSet):
for micU, micT in izip(uSet, tSet):
coordFnU, coordFnT = self.getMatchingCoord(micU, micT)
if coordFnU and coordFnT:
def addCoordinateU(coord):
coord.setMicrograph(micU)
self.correctCoordinatePosition(coord)
uCoordSet.append(coord)
ci.importCoordinates(coordFnU, addCoordinate=addCoordinateU)
def addCoordinateT(coord):
coord.setMicrograph(micT)
self.correctCoordinatePosition(coord)
tCoordSet.append(coord)
ci.importCoordinates(coordFnT, addCoordinate=addCoordinateT)
def addAngles(ang):
anglesSet.append(ang)
if self.importFrom.get() == self.IMPORT_FROM_EMAN:
ci.importAngles(coordFnT, addAngles=addAngles)
_importCoords(uCoordSet, tCoordSet)
boxSize = self.boxSize.get()
uCoordSet.setBoxSize(boxSize)
tCoordSet.setBoxSize(boxSize)
uCoordSet.write()
tCoordSet.write()
if self.importFrom.get() == self.IMPORT_FROM_XMIPP:
readAnglesFromMicrographs = Domain.importFromPlugin('xmipp3.convert',
'readAnglesFromMicrographs')
if exists(self.xmippMdFn.get()):
checkAngles = self._compareMicPairs(self.xmippMdFn, uSet, tSet)
if checkAngles:
readAnglesFromMicrographs(self.xmippMdFn, anglesSet)
else:
self.warning('Angles for some micrographs were not found, '
'skipping angles import')
else:
self.warning('Micrograph xmd file not provided, so tilt angles will '
'not be imported')
anglesSet.write()
return uCoordSet, tCoordSet, anglesSet
def convertStack(self,
inputFn,
outputFn,
firstImg=None,
lastImg=None,
inFormat=None,
outFormat=None):
""" Convert an input stack file into another.
It is possible to only use a subset of frames to be written in the
output stack.
If outFormat/inFomat=None then there will be
inferred from extension.If firstFrame/lastFrame are not None, the output
stack will be a subset of input stack. If it are none, the conversion is
over the whole stack. If the input format is ".dm4" or ".img" only is
allowed the conversion of the whole stack.
"""
# inputLower = inputFn.lower()
outputLower = outputFn.lower()
if outputLower.endswith('.img'):
if (firstImg and lastImg) is None:
# FIXME Since now we can not read dm4 format in Scipion natively
# or writing recent .img format
# we are opening an Eman2 process to read the dm4 file
from pwem import Domain
convertImage = Domain.importFromPlugin('eman2.convert',
'convertImage')
convertImage(inputFn, outputFn)
else:
ext = os.path.splitext(outputFn)[1]
raise Exception("if convert from %s, firstImg and lastImg "
"must be None" % ext)
# elif inputLower.endswith('.tif'):
# # FIXME: It seems that we have some flip problem with compressed
# # tif files, we need to check that
# if outputLower.endswith('.mrc'):
# self.runJob('tif2mrc', '%s %s' % (inputFn, outputFn))
# else:
# raise Exception("Conversion from tif to %s is not "
# "implemented yet. " % pwutils.getExt(outputFn))
else:
# get input dim
(x, y, z, n) = lib.getImageSize(inputFn)
location = self._convertToLocation(inputFn)
self._img.read(location, lib.HEADER)
dataType = self.getSupportedDataType(self._img.getDataType(),
outputLower)
if (firstImg and lastImg) is None:
n = max(z, n)
firstImg = 1
lastImg = n
else:
n = lastImg - firstImg + 1
# Create empty output stack file to reserve desired space
lib.createEmptyFile(outputFn, x, y, 1, n, dataType)
for i, j in zip(range(firstImg, lastImg + 1), range(1, n + 1)):
self.convert((i, inputFn), (j, outputFn))
def test1(self):
# First, import a set of micrographs
protImport = self.newProtocol(emprot.ProtImportMicrographsTiltPairs,
patternUntilted=self.micsUFn,
patternTilted=self.micsTFn,
samplingRate=2.28,
voltage=100,
sphericalAberration=2.9)
self.launchProtocol(protImport)
self.assertIsNotNone(protImport.outputMicrographsTiltPair,
"There was a problem with the import")
protImportCoorU = self.newProtocol(
emprot.ProtImportCoordinates,
importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_XMIPP,
filesPath=self.allCrdsDir,
filesPattern='F_rct_u_*.pos',
boxSize=100)
uMics = protImport.outputMicrographsTiltPair.getUntilted()
protImportCoorU.inputMicrographs.set(uMics)
self.launchProtocol(protImportCoorU)
tMics = protImport.outputMicrographsTiltPair.getTilted()
protImportCoorT = self.newProtocol(
emprot.ProtImportCoordinates,
importFrom=emprot.ProtImportCoordinates.IMPORT_FROM_XMIPP,
filesPath=self.allCrdsDir,
filesPattern='F_rct_t_*.pos',
boxSize=100)
protImportCoorT.inputMicrographs.set(tMics)
self.launchProtocol(protImportCoorT)
# Then simulate a particle picking
print("Running tilt pairs assignment...")
XmippProtAssignmentTiltPair = Domain.importFromPlugin(
'xmipp3.protocols', 'XmippProtAssignmentTiltPair', doRaise=True)
protAssigning = self.newProtocol(XmippProtAssignmentTiltPair)
micsTiltPair = protImport.outputMicrographsTiltPair
protAssigning.inputMicrographsTiltedPair.set(micsTiltPair)
print(self.micsUFn)
print(self.micsTFn)
protAssigning.untiltedSet.set(protImportCoorU.outputCoordinates)
protAssigning.tiltedSet.set(protImportCoorT.outputCoordinates)
self.launchProtocol(protAssigning)
self.assertIsNotNone(
protAssigning.outputCoordinatesTiltPair,
"There was a problem with the protocol assignment tilt pairs")
print('-----------------------------------------------------------')
num_particles = protAssigning.outputCoordinatesTiltPair.getUntilted(
).getSize()
print(num_particles)
if num_particles > 1000:
out_ = True
else:
out_ = None
self.assertIsNotNone(
out_,
"There was a problem with the protocol assignment tilt pairs")
def sleepStep(self, secs=5, forceFail=False, tag=''):
if forceFail:
self.runJob('echo', " 'Failing for testing purposes...'; exit 1")
else:
from pwem import Domain
getEnviron = Domain.importFromPlugin('xmipp3', 'getEnviron')
self.runJob('xmipp_work_test',
"--time %d --tag '%s'" % (secs, tag), env=getEnviron())
def _readAngles(self, micsFn, suffix=''):
# Read Angles from input micrographs
anglesSet = self._createSetOfAngles(suffix=suffix)
readAnglesFromMicrographs = Domain.importFromPlugin('xmipp3.convert',
'readAnglesFromMicrographs')
readAnglesFromMicrographs(micsFn, anglesSet)
anglesSet.write()
return anglesSet
def __getXmippEulerAngles(matrix):
""" Internal fuction to convert scipion to xmipp angles"""
from pwem import Domain
geometryFromMatrix = Domain.importFromPlugin('xmipp3.convert',
'geometryFromMatrix',
doRaise=True)
return geometryFromMatrix(matrix, False)
def getImportClass(self):
""" Return the class in charge of importing the files. """
if self.importFrom == self.IMPORT_FROM_EMX:
EmxImport = Domain.importFromPlugin('emxlib.convert', 'EmxImport')
self.importFilePath = self.emxFile.get('').strip()
return EmxImport(self, self.importFilePath)
elif self.importFrom == self.IMPORT_FROM_XMIPP3:
XmippImport = Domain.importFromPlugin('xmipp3.convert',
'XmippImport')
self.importFilePath = self.mdFile.get('').strip()
return XmippImport(self, self.mdFile.get())
elif self.importFrom == self.IMPORT_FROM_SCIPION:
from .dataimport import ScipionImport
self.importFilePath = self.sqliteFile.get('').strip()
return ScipionImport(self, self.importFilePath)
else:
self.importFilePath = ''
return None
def __runXmippProgram(program, args):
""" Internal shortcut function to launch a Xmipp program.
If xmipp not available o fails return False, else Tru"""
try:
from pwem import Domain
xmipp3 = Domain.importFromPlugin('xmipp3', doRaise=True)
xmipp3.Plugin.runXmippProgram(program, args)
except ImportError:
return False
return True
def runNormalizeParticles(cls, particles):
""" Run normalize particles protocol """
relionProtocols = Domain.importFromPlugin('relion.protocols',
doRaise=True)
protPreproc = cls.newProtocol(relionProtocols.ProtRelionPreprocessParticles,
doNormalize=True)
protPreproc.inputParticles.set(particles)
cls.launchProtocol(protPreproc)
cls.sampling = protPreproc.outputParticles.getSamplingRate()
return protPreproc
def _getMovieGainProtocol(self):
XmippProtMovieGain = Domain.importFromPlugin('xmipp3.protocols',
'XmippProtMovieGain')
if XmippProtMovieGain is None:
return None
for protPointer in self.inputProtocols:
prot = protPointer.get()
if prot.getClassName() == XmippProtMovieGain.__name__:
return prot
return None
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
kwargs = {
'xDim': 4096,
'yDim': 4096,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 5,
'delay': 0,
'setof': emprot.SET_OF_RANDOM_MICROGRAPHS # SetOfMicrographs
}
# put some stress on the system
protStream = self.newProtocol(emprot.ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
self._waitOutput(protStream, 'outputMicrographs')
# then introduce monitor, checking all the time ctf and saving to
# database
kwargs = {
'useCtffind4': True,
'ctfDownFactor': 2,
'numberOfThreads': 4
}
from pwem import Domain
ProtCTFFind = Domain.importFromPlugin('cistem.protocols',
'CistemProtCTFFind',
doRaise=True)
protCTF = self.newProtocol(ProtCTFFind, **kwargs)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
self.proj.launchProtocol(protCTF, wait=False)
self._waitOutput(protCTF, 'outputCTF')
kwargs = {
'samplingInterval': 10,
'interval': 300,
'maxDefocus': 40000,
'minDefocus': 1000,
'astigmatism': 0.2,
'monitorTime': 5
}
protMonitor = self.newProtocol(monitorsProt.ProtMonitorCTF, **kwargs)
protMonitor.inputProtocol.set(protCTF)
self.launchProtocol(protMonitor)
baseFn = protMonitor._getPath(monitorsProt.CTF_LOG_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
def _displayMapModel(self, e=None):
bildFileName = self.protocol._getExtraPath("axis_output.bild")
_inputVol = self.protocol.inputVolume.get()
if _inputVol is None:
_inputVol = self.protocol.inputStructure.get().getVolume()
dim = _inputVol.getDim()[0]
sampling = _inputVol.getSamplingRate()
Chimera.createCoordinateAxisFile(dim,
bildFileName=bildFileName,
sampling=sampling)
counter = 1
fnCmd = self.protocol._getExtraPath("chimera_output.cxc")
f = open(fnCmd, 'w')
# change to workingDir
# If we do not use cd and the project name has an space
# the protocol fails even if we pass absolute paths
f.write('cd %s\n' % os.getcwd())
# reference axis model = 0
f.write("open %s\n" % bildFileName)
f.write("cofr 0,0,0\n") # set center of coordinates
# input 3D map
counter += 1 # 1
fnVol = self._getInputVolume()
if fnVol is not None:
EMRINGERFILENAME = self.protocol._getExtraPath(
self.protocol.EMRINGERFILE)
f.write("open %s\n" % EMRINGERFILENAME)
x, y, z = fnVol.getOrigin(force=True).getShifts()
sampling = fnVol.getSamplingRate()
f.write("volume #%d style surface voxelSize %f\nvolume #%d origin "
"%0.2f,%0.2f,%0.2f\n" %
(counter, sampling, counter, x, y, z))
# input PDB (usually from coot)
counter += 1 # 2
pdbFileName = self.protocol.inputStructure.get().getFileName()
f.write("open %s\n" % pdbFileName)
f.close()
# run in the background
chimeraPlugin = Domain.importFromPlugin('chimera',
'Plugin',
doRaise=True)
chimeraPlugin.runChimeraProgram(chimeraPlugin.getProgram(),
fnCmd + "&",
cwd=os.getcwd())
return []
def runProgram(program, params):
""" Runs a em program setting its environment matching a prefix"""
env = None
# Allow passing absolute paths
programName = basename(program)
from pwem import Domain
if programName.startswith('xmipp'):
xmipp3 = Domain.getPlugin('xmipp3').Plugin
env = xmipp3.getEnviron()
if programName.startswith('relion'):
relion = Domain.getPlugin("relion").Plugin
env = relion.getEnviron()
print(env)
elif (programName.startswith('e2') or programName.startswith('sx')):
eman2 = Domain.importFromPlugin('eman2', 'Plugin')
env = eman2.getEnviron()
elif programName.startswith('b'):
bsoft = Domain.importFromPlugin('bsoft', 'Plugin')
env = bsoft.getEnviron()
pwutils.runJob(None, program, params, env=env)
def _computePSD(self, inputFn, outputFn, scaleFactor=6):
""" Generate a thumbnail of the PSD with EMAN2"""
args = "%s %s " % (inputFn, outputFn)
args += "--process=math.realtofft --meanshrink %s " % scaleFactor
args += "--fixintscaling=sane"
from pwem import Domain
eman2 = Domain.importFromPlugin('eman2')
from pyworkflow.utils.process import runJob
runJob(self._log,
eman2.Plugin.getProgram('e2proc2d.py'),
args,
env=eman2.Plugin.getEnviron())
return outputFn