def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Prepare environment
from appion import Plugin
Plugin.getEnviron()
# Program to execute and it arguments
program = "python2"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(
inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
dogpicker = Plugin.getHome("ApDogPicker.py")
args = dogpicker + " " + args
self.runJob(program, args)
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameter just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
imgStarTmp = self._getTmpPath('movie_particles.star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStarTmp))
writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(),
alignType=imgSet.getAlignment(),
extraLabels=MOVIE_EXTRA_LABELS)
mdImg = md.MetaData(imgStarTmp)
# replace mdColumn from *.stk to *.mrcs as Relion2 requires
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
from convert import relionToLocation, locationToRelion
for objId in mdImg:
index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId))
if not imgPath.endswith('mrcs'):
newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs')
newPath = self._getTmpPath(newName)
newLoc = locationToRelion(index, newPath)
if not exists(newPath):
pwutils.createLink(imgPath, newPath)
mdImg.setValue(mdColumn, newLoc, objId)
mdImg.write(imgStar)
def createBinaryLink(fn):
""" Just create a link named .mrcs to Relion understand
that it is a binary stack file and not a volume.
"""
newFn = getUniqueFileName(fn, extension)
pwutils.createLink(fn, newFn)
return newFn
def _inputVol2Mrc(self, inputFname, outputFname):
if inputFname.endswith(".mrc") or inputFname.endswith(".map"):
if not os.path.exists(outputFname):
createLink(inputFname, outputFname)
else:
self.runJob('xmipp_image_convert',
" -i %s -o %s:mrc -t vol" % (inputFname, outputFname))
def convertInputStep(self, micsId, refsId):
""" This step will take of the convertions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrc' format
Mask: either converted ('.tif' format) or generated a circular one
"""
ih = em.ImageHandler()
micDir = self._getTmpPath('micrographs')
pwutils.makePath(micDir)
for mic in self.getInputMicrographs():
# Create micrograph folder
micName = mic.getFileName()
# If micrographs are in .mrc format just link it
# otherwise convert them
outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(mic, outMic)
refDir = self._getTmpPath('templates')
pwutils.makePath(refDir)
# We will always convert the templates, since
# they can be in an stack and link will not be possible sometimes
inputRefs = self.inputReferences.get()
for i, ref in enumerate(inputRefs):
outRef = join(refDir, 'ref%02d.mrc' % (i + 1))
ih.convert(ref, outRef)
self.createInputMasks(inputRefs)
def _mergeDataStar(self, outStar, mdInput, iter, rLev):
imgStar = self._getFileName('data',
iter=iter,
lev=self._level,
rLev=rLev)
mdData = md.MetaData(imgStar)
print("reading %s and begin the loop" % imgStar)
for row in md.iterRows(mdData, sortByLabel=md.RLN_PARTICLE_CLASS):
clsPart = row.getValue(md.RLN_PARTICLE_CLASS)
if clsPart != self._lastCls:
self._newClass += 1
self._clsDict['%s.%s' % (rLev, clsPart)] = self._newClass
self._lastCls = clsPart
# write symlink to new Maps
relionFn = self._getFileName('relionMap',
lev=self._level,
iter=self._getnumberOfIters(),
ref3d=clsPart,
rLev=rLev)
newFn = self._getFileName('map',
lev=self._level,
rLev=self._newClass)
print(('link from %s to %s' % (relionFn, newFn)))
createLink(relionFn, newFn)
row.setValue(md.RLN_PARTICLE_CLASS, self._newClass)
row.addToMd(mdInput)
print("writing %s and ending the loop" % outStar)
def classifyStep(self):
""" Run MSA-CL and MSA-SUM from IMAGIC. """
inputFile = self.inputMSA.get().getParticlesStack()
inputFileBase = pwutils.removeExt(inputFile)
inputFileImg = inputFileBase + '.img'
inputFileHed = inputFileBase + '.hed'
pwutils.createLink(inputFileImg, self._getTmpPath("particles.img"))
pwutils.createLink(inputFileHed, self._getTmpPath("particles.hed"))
inputFn = "tmp/particles"
if self.doDownweight.get():
downweight = 'YES'
else:
downweight = 'NO'
self._params.update({
'particles': inputFn,
'eigs_num': self.numberOfFactors.get(),
'cls_num': self.numberOfClasses.get(),
'perc_ign': self.percentIgnore.get(),
'downweight': downweight,
'perc_ign_bad': self.percentIgnoreBad.get()
})
classDir = self._getPath(self.CLASS_DIR)
pwutils.cleanPath(classDir)
pwutils.makePath(classDir)
self.runTemplate('msa/msa-cls.b', self._params)
def importParticles(self):
""" Import particles from Frealign.
Params:
parFile: the filename of the parameter file with the alignment in Frealign.
stackFile: single stack file with the images.
"""
partSet = self.protocol._createSetOfParticles()
partSet.setObjComment('Particles imported from Frealign parfile:\n%s' %
self.parFile)
# Create a local link to the input stack file
localStack = self.protocol._getExtraPath(
os.path.basename(self.stackFile))
pwutils.createLink(self.stackFile, localStack)
# Create a temporary set only with location
tmpSet = SetOfParticles(filename=':memory:')
tmpSet.readStack(localStack)
self._setupSet(tmpSet)
# Update both samplingRate and acquisition with parameters
# selected in the protocol form
self._setupSet(partSet)
# Now read the alignment parameters from par file
readSetOfParticles(tmpSet, partSet, self.parFile)
partSet.setHasCTF(True)
# Register the output set of particles
self.protocol._defineOutputs(outputParticles=partSet)
def retrieveTrainSets(self):
""" Retrieve, link and return a setOfParticles
corresponding to the NegativeTrain DeepConsensus trainning set
with certain extraction conditions (phaseFlip/invContrast)
"""
prefixYES = ''
prefixNO = 'no'
modelType = "negativeTrain_%sPhaseFlip_%sInvert.mrcs" % (
prefixYES if self.doInvert.get() else prefixNO,
prefixYES if self.ignoreCTF.get() else prefixNO)
modelPath = xmipp3.Plugin.getModel("deepConsensus", modelType)
print("Precompiled negative particles found at %s" % (modelPath))
modelFn = self._getTmpPath(modelType)
pwutils.createLink(modelPath, modelFn)
tmpSqliteSuff = "AddTrain"
partSet = self._createSetOfParticles(tmpSqliteSuff)
img = SetOfParticles.ITEM_TYPE()
imgh = ImageHandler()
_, _, _, n = imgh.getDimensions(modelFn)
if n > 1:
for index in range(1, n + 1):
img.cleanObjId()
img.setMicId(9999)
img.setFileName(modelFn)
img.setIndex(index)
partSet.append(img)
partSet.setAlignment(ALIGN_NONE)
cleanPath(self._getPath("particles%s.sqlite" % tmpSqliteSuff))
return partSet
def convertInputStep(self, micsId, refsId):
""" This step will take of the convertions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrc' format
Mask: either converted ('.tif' format) or generated a circular one
"""
ih = em.ImageHandler()
micDir = self._getTmpPath('micrographs')
pwutils.makePath(micDir)
for mic in self.getInputMicrographs():
# Create micrograph folder
micName = mic.getFileName()
# If micrographs are in .mrc format just link it
# otherwise convert them
outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(mic, outMic)
refDir = self._getTmpPath('templates')
pwutils.makePath(refDir)
# We will always convert the templates, since
# they can be in an stack and link will not be possible sometimes
inputRefs = self.inputReferences.get()
for i, ref in enumerate(inputRefs):
outRef = join(refDir, 'ref%02d.mrc' % (i+1))
ih.convert(ref, outRef)
self.createInputMasks(inputRefs)
def importParticles(self):
""" Import particles from Frealign.
Params:
parFile: the filename of the parameter file with the alignment in Frealing.
stackFile: single stack file with the images.
"""
partSet = self.protocol._createSetOfParticles()
partSet.setObjComment("Particles imported from Frealign parfile:\n%s" % self.parFile)
# Create a local link to the input stack file
localStack = self.protocol._getExtraPath(os.path.basename(self.stackFile))
pwutils.createLink(self.stackFile, localStack)
# Create a temporarly set only with location
tmpSet = SetOfParticles(filename=":memory:")
tmpSet.readStack(localStack)
self._setupSet(tmpSet)
# Update both samplingRate and acquisition with parameters
# selected in the protocol form
self._setupSet(partSet)
# Now read the alignment parameters from par file
readSetOfParticles(tmpSet, partSet, self.parFile)
partSet.setHasCTF(True)
# Register the output set of particles
self.protocol._defineOutputs(outputParticles=partSet)
def exportWorkflow(self):
project = self.getProject()
workflowProts = [p for p in project.getRuns()
] # workflow prots are all prots if no json provided
workflowJsonPath = os.path.join(
project.path, self.getTopLevelPath(self.OUTPUT_WORKFLOW))
protDicts = project.getProtocolDicts(workflowProts)
for inputSetPointer in self.inputSets:
inputSet = inputSetPointer.get()
setName = inputSet.getObjName()
setParentId = inputSet.getObjParentId()
setParentObj = project.getObject(setParentId)
protDicts[setParentId]['filesPath'] = os.path.join('.', setName)
pwutils.createLink(setParentObj._getExtraPath(),
self.getTopLevelPath(setName))
with open(workflowJsonPath, 'w') as f:
f.write(
json.dumps(list(protDicts.values()),
indent=4,
separators=(',', ': ')))
self.workflowPath.set(workflowJsonPath)
return workflowJsonPath
def _initialize(self):
# The prediction is expecting the training and validation datasets to be in the same place as the training
# model, but they are located in the training data generation extra directory. Hence, a symbolic link will
# be created
makeDatasetSymLinks(self, self.trainDataDir.get())
createLink(join('..', self.basedir.get()),
self._getExtraPath(CRYOCARE_MODEL))
def writeCtfStarStep(self):
inputCTF = self.inputCTF.get()
if self.micrographSource == 0: # same as CTF estimation
ctfMicSet = inputCTF.getMicrographs()
else:
ctfMicSet = self.inputMicrographs.get()
micSet = SetOfMicrographs(filename=':memory:')
psd = inputCTF.getFirstItem().getPsdFile()
hasPsd = psd and os.path.exists(psd)
if hasPsd:
psdPath = self._getPath('PSD')
pwutils.makePath(psdPath)
print "Writing PSD files to %s" % psdPath
for ctf in inputCTF:
# Get the corresponding micrograph
mic = ctfMicSet[ctf.getObjId()]
if mic is None:
print("Skipping CTF id: %s, it is missing from input "
"micrographs. " % ctf.getObjId())
continue
micFn = mic.getFileName()
if not os.path.exists(micFn):
print "Skipping micrograph %s, it does not exists. " % micFn
continue
mic2 = mic.clone()
mic2.setCTF(ctf)
if hasPsd:
psdFile = ctf.getPsdFile()
newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
if not os.path.exists(psdFile):
print "PSD file %s does not exits" % psdFile
print "Skipping micrograph %s" % micFn
continue
pwutils.copyFile(psdFile, newPsdFile)
ctf.setPsdFile(newPsdFile)
micSet.append(mic2)
starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
print "Writing set: %s" % inputCTF
print " to: %s" % starFile
acq = ctfMicSet.getAcquisition()
self.samplingRate = ctfMicSet.getSamplingRate()
mag = acq.getMagnification()
self.detectorPixelSize = 1e-4 * self.samplingRate * mag
writeSetOfMicrographs(micSet, starFile,
preprocessImageRow=self.preprocessMicrograph)
# Let's create a link from the project roots to facilitate the import
# of the star file into a Relion project
pwutils.createLink(starFile, os.path.basename(starFile))
def createBinaryLink(fn):
""" Just create a link named .mrcs to Relion understand
that it is a binary stack file and not a volume.
"""
newFn = getUniqueFileName(fn, extension)
if not os.path.exists(newFn):
pwutils.createLink(fn, newFn)
print(" %s -> %s" % (newFn, fn))
return newFn
def createLinksStep(self):
prot = self._inputProt()
prevPartDir = prot._getExtraPath("particles")
currPartDir = self._getExtraPath("particles")
prevSetsDir = prot._getExtraPath("sets")
currSetsDir = self._getExtraPath("sets")
createLink(prevPartDir, currPartDir)
createLink(prevSetsDir, currSetsDir)
def importMicrographs(self, pattern, suffix, voltage, sphericalAberration,
amplitudeContrast):
""" Copy images matching the filename pattern
Register other parameters.
"""
filePaths = glob(pwutils.expandPattern(pattern))
# imgSet = SetOfMicrographs(filename=self.micsPairsSqlite, prefix=suffix)
imgSet = self._createSetOfMicrographs(suffix=suffix)
acquisition = imgSet.getAcquisition()
# Setting Acquisition properties
acquisition.setVoltage(voltage)
acquisition.setSphericalAberration(sphericalAberration)
acquisition.setAmplitudeContrast(amplitudeContrast)
# Call a function that should be implemented by each subclass
self._setOtherPars(imgSet)
outFiles = [imgSet.getFileName()]
imgh = emlib.image.ImageHandler()
img = imgSet.ITEM_TYPE()
n = 1
size = len(filePaths)
filePaths.sort()
for i, fn in enumerate(filePaths):
# ext = os.path.splitext(basename(f))[1]
dst = self._getExtraPath(basename(fn))
if self.copyToProj:
pwutils.copyFile(fn, dst)
else:
pwutils.createLink(fn, dst)
if n > 1:
for index in range(1, n + 1):
img.cleanObjId()
img.setFileName(dst)
img.setIndex(index)
imgSet.append(img)
else:
img.cleanObjId()
img.setFileName(dst)
# Fill the micName if img is a Micrograph.
self._fillMicName(img, fn, pattern)
imgSet.append(img)
outFiles.append(dst)
sys.stdout.write("\rImported %d/%d" % (i + 1, size))
sys.stdout.flush()
print("\n")
imgSet.write()
return imgSet
def writeCtfStarStep(self):
inputCTF = self.inputCTF.get()
if self.micrographSource == 0: # same as CTF estimation
ctfMicSet = inputCTF.getMicrographs()
else:
ctfMicSet = self.inputMicrographs.get()
micSet = SetOfMicrographs(filename=':memory:')
psd = inputCTF.getFirstItem().getPsdFile()
hasPsd = psd and os.path.exists(psd)
if hasPsd:
psdPath = self._getPath('PSD')
pwutils.makePath(psdPath)
print "Writing PSD files to %s" % psdPath
for ctf in inputCTF:
# Get the corresponding micrograph
mic = ctfMicSet[ctf.getObjId()]
if mic is None:
print(
"Skipping CTF id: %s, it is missing from input "
"micrographs. " % ctf.getObjId())
continue
micFn = mic.getFileName()
if not os.path.exists(micFn):
print "Skipping micrograph %s, it does not exists. " % micFn
continue
mic2 = mic.clone()
mic2.setCTF(ctf)
if hasPsd:
psdFile = ctf.getPsdFile()
newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
if not os.path.exists(psdFile):
print "PSD file %s does not exits" % psdFile
print "Skipping micrograph %s" % micFn
continue
pwutils.copyFile(psdFile, newPsdFile)
ctf.setPsdFile(newPsdFile)
micSet.append(mic2)
starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
print "Writing set: %s" % inputCTF
print " to: %s" % starFile
writeSetOfMicrographs(micSet,
starFile,
preprocessImageRow=self.preprocessMicrograph)
# Let's create a link from the project roots to facilitate the import
# of the star file into a Relion project
pwutils.createLink(starFile, os.path.basename(starFile))
def convertImagesStep(self):
partSet = self._getInputParticles()
partAlign = partSet.getAlignment()
storePath = self._getExtraPath("particles")
makePath(storePath)
writeSetOfParticles(partSet, storePath, alignType=partAlign)
if self.useInputBispec and self.inputBispec is not None:
print(
"Skipping CTF estimation since input bispectra were provided")
self.skipctf.set(True)
if not self.skipctf:
program = eman2.Plugin.getProgram('e2ctf.py')
acq = partSet.getAcquisition()
args = " --voltage %d" % acq.getVoltage()
args += " --cs %f" % acq.getSphericalAberration()
args += " --ac %f" % (100 * acq.getAmplitudeContrast())
args += " --threads=%d" % self.numberOfThreads.get()
if not partSet.isPhaseFlipped():
args += " --phaseflip"
args += " --computesf --apix %f" % partSet.getSamplingRate()
args += " --allparticles --autofit --curdefocusfix --storeparm -v 8"
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
program = eman2.Plugin.getProgram('e2buildsets.py')
args = " --setname=inputSet --allparticles --minhisnr=-1"
self.runJob(program,
args,
cwd=self._getExtraPath(),
numberOfMpi=1,
numberOfThreads=1)
if self.useInputBispec:
prot = self.inputBispec.get()
prot._createFilenameTemplates()
bispec = prot.outputParticles_flip_bispec
if not bispec.getSize() == partSet.getSize():
raise Exception(
'Input particles and bispectra sets have different size!')
# link bispec hdf files and lst file
pattern = prot._getExtraPath('particles/*__ctf_flip_bispec.hdf')
print("\nLinking bispectra input files...")
for fn in sorted(glob(pattern)):
newFn = join(self._getExtraPath('particles'), basename(fn))
createLink(fn, newFn)
print(" %s -> %s" % (fn, newFn))
lstFn = prot._getFileName('partSetFlipBispec')
newLstFn = self._getFileName('partBispecSet')
createLink(lstFn, newLstFn)
def writeSetOfVolumes(volSet, volXml, volDir):
""" Convert a SetOfVolumes to a xml file used by PyTom.
The volumes will be converted to .mrc format if not are '.em' or '.mrc'
Params:
volSet: input SetOfVolumes.
volXml: filename where to write the xml file.
volDir: where to create links or copies (converted to mrc).
"""
# Add to the path the root to pytom
backupPath = list(sys.path)
addPyTomPaths()
from pytom.basic.structures import Particle, ParticleList, Wedge, SingleTiltWedge
from pytom.score.score import Score, PeakPrior, xcfScore
from pytom.frm.FRMAlignment import FRMScore
w = SingleTiltWedge()
#s = PeakPrior()
pl = ParticleList()
ih = em.convert.ImageHandler()
for vol in volSet:
index, fn = vol.getLocation()
convert = True # by default, convert, which is the save way
if index == em.NO_INDEX: # means single volumes
volName = os.path.basename(fn)
if fn.endswith('.em') or fn.endswith('.mrc'):
convert = False # we can just create a link in this case
else:
volName = 'volume_%03d.mrc' % vol.getObjId()
volFn = os.path.join(volDir, volName)
if convert:
ih.convert(vol, volFn)
else:
pwutils.createLink(fn, volFn)
# Make the volumes names relative to the xml file
# where the programs will be executed
volRel = os.path.relpath(volFn, os.path.dirname(volXml))
p = Particle()
s = xcfScore()
s.setValue(1.0)
pytomInfo = getattr(vol, 'pytomInfo', None)
if pytomInfo is None:
p.setWedge(w)
else:
p.fromXML(pytomInfo.get()) # Get stored XML format from PyTom
p.setFilename(volRel)
p.setScore(s)
pl.append(p)
pl.toXMLFile(volXml)
#pl.setWedgeAllParticles(w)
sys.path = backupPath
def _visualize(self, obj, **args):
cls = type(obj)
if issubclass(cls, Volume) or issubclass(cls, Image):
fn = obj.getFileName()
if pwutils.getExt(fn) in ['.stk', '.vol', '.xmp']:
# eman2 expects spider files with .spi extension only
pwutils.createLink(fn, pwutils.replaceExt(fn, 'spi'))
fn = pwutils.replaceExt(fn, 'spi')
view = CommandView("%s %s &" %
(Plugin.getProgram('e2display.py'), fn),
env=Plugin.getEnviron())
return [view]
elif isinstance(obj, EmanProtBoxing):
coords = obj.getCoords()
if coords:
return DataViewer._visualize(self, obj.outputCoordinates)
elif isinstance(obj, EmanProtInitModel):
obj = obj.outputVolumes
fn = obj.getFileName()
labels = 'id enabled comment _filename '
objCommands = "'%s' '%s' '%s'" % (OBJCMD_CLASSAVG_PROJS,
OBJCMD_PROJS, OBJCMD_INITVOL)
self._views.append(
ObjectView(self._project,
obj.strId(),
fn,
viewParams={
showj.MODE: showj.MODE_MD,
showj.VISIBLE: labels,
showj.RENDER: '_filename',
showj.OBJCMDS: objCommands
}))
return self._views
elif isinstance(obj, EmanProtInitModelSGD):
obj = obj.outputVolumes
fn = obj.getFileName()
labels = 'id enabled comment _filename '
objCommands = "'%s'" % OBJCMD_CLASSAVG_PROJS
self._views.append(
ObjectView(self._project,
obj.strId(),
fn,
viewParams={
showj.MODE: showj.MODE_MD,
showj.VISIBLE: labels,
showj.RENDER: '_filename',
showj.OBJCMDS: objCommands
}))
return self._views
def createBsoftInputParticles(imgSet, starFile, stackFile):
""" Ensure that 'starFile' is a valid STAR files with particles.
If the imgSet comes from Bsoft, just create a link.
If not, then write the proper file.
"""
imgsStar = getattr(imgSet, '_bsoftStar', None)
if imgsStar is None:
writeSetOfParticles(imgSet, starFile, stackFile)
else:
imgsFn = imgsStar.get()
createLink(imgsFn, imgsStar.get())
def copyOrLinkFileName(imgRow, prefixDir, outputDir, copyFiles=False):
index, imgPath = relionToLocation(imgRow.getValue(md.RLN_IMAGE_NAME))
baseName = os.path.basename(imgPath)
newName = os.path.join(outputDir, baseName)
if not os.path.exists(newName):
if copyFiles:
pwutils.copyFile(os.path.join(prefixDir, imgPath), newName)
else:
pwutils.createLink(os.path.join(prefixDir, imgPath), newName)
imgRow.setValue(md.RLN_IMAGE_NAME, locationToRelion(index, newName))
def create_resmap_project(request):
if request.is_ajax():
import os
from pyworkflow.em.protocol import ProtImportVolumes
from pyworkflow.em.packages.resmap.protocol_resmap import ProtResMap
# Create a new project
projectName = request.GET.get(PROJECT_NAME)
# Filename to use as test data
testDataKey = request.GET.get('testData')
manager = getServiceManager(MYRESMAP_SERVICE)
writeCustomMenu(manager.protocols)
project = manager.createProject(projectName,
runsView=1,
hostsConf=manager.hosts,
protocolsConf=manager.protocols,
chdir=False)
project.getSettings().setLifeTime(336) # 14 days * 24 hours
project.saveSettings()
projectPath = manager.getProjectPath(projectName)
# 1. Import maps
if testDataKey:
attr = getAttrTestFile(testDataKey)
source = attr['file']
dest = os.path.join(projectPath, 'Uploads', basename(source))
pwutils.createLink(source, dest)
label_import = "import volumes (" + testDataKey + ")"
protImport = project.newProtocol(ProtImportVolumes,
objLabel=label_import)
protImport.filesPath.set(dest)
protImport.samplingRate.set(attr['samplingRate'])
project.launchProtocol(protImport, wait=True, chdir=False)
else:
protImport = project.newProtocol(ProtImportVolumes,
objLabel='import volumes')
project.saveProtocol(protImport)
# 2. ResMap
protResMap = project.newProtocol(ProtResMap)
protResMap.setObjLabel('resmap - local resolution')
protResMap.inputVolume.set(protImport)
protResMap.inputVolume.setExtended('outputVolume')
loadProtocolConf(protResMap)
project.saveProtocol(protResMap)
return HttpResponse(content_type='application/javascript')
def __init__(self, srcPath, dstPath, copyFiles=False):
self._src = srcPath
self._dst = dstPath
self._processed = set()
if copyFiles:
pwutils.makePath(dstPath)
self.transform = self._copyFile
else:
pwutils.makePath(os.path.dirname(srcPath))
pwutils.createLink(srcPath, dstPath)
self.transform = self._addPrefix
def create_resmap_project(request):
if request.is_ajax():
import os
from pyworkflow.object import Pointer
from pyworkflow.em.protocol import ProtImportVolumes
from pyworkflow.em.packages.resmap.protocol_resmap import ProtResMap
# Create a new project
projectName = request.GET.get('projectName')
# Filename to use as test data
testDataKey = request.GET.get('testData')
manager = getServiceManager('myresmap')
writeCustomMenu(manager.protocols)
project = manager.createProject(projectName, runsView=1,
hostsConf=manager.hosts,
protocolsConf=manager.protocols
)
project.getSettings().setLifeTime(14)
project.saveSettings()
projectPath = manager.getProjectPath(projectName)
# 1. Import movies
if testDataKey :
attr = getAttrTestFile(testDataKey)
source = attr['path'] + attr['file']
dest = os.path.join(projectPath,'Uploads', attr['file'])
pwutils.createLink(source, dest)
label_import = "import volumes ("+ testDataKey +")"
protImport = project.newProtocol(ProtImportVolumes, objLabel=label_import)
protImport.filesPath.set(dest)
protImport.samplingRate.set(attr['samplingRate'])
project.launchProtocol(protImport, wait=True)
else:
protImport = project.newProtocol(ProtImportVolumes, objLabel='import volumes')
project.saveProtocol(protImport)
# 2. ResMap
protResMap = project.newProtocol(ProtResMap)
protResMap.setObjLabel('resmap - local resolution')
protResMap.inputVolume.set(protImport)
protResMap.inputVolume.setExtendedAttribute('outputVolume')
project.saveProtocol(protResMap)
return HttpResponse(mimetype='application/javascript')
def _convertInputStep(self):
self.info("Relion version:")
self.runJob("relion_convert_to_tiff --version", "", numberOfMpi=1)
self.info("Detected version from config: %s" %
relion.Plugin.getActiveVersion())
# Create a local link to the input gain file if necessary
inputGain = self.getInputGain()
if inputGain:
tmpGain = self._getTmpPath('gain_estimate.bin')
pwutils.createLink(inputGain, tmpGain)
pwutils.createLink(inputGain.replace('.bin', '_reliablity.bin'),
tmpGain.replace('.bin', '_reliablity.bin'))
ProtAlignMovies._convertInputStep(self)
def createOutputStep(self):
imgSet = self.inputParticles.get()
# Let us use extension "vol" for the output vol
# use stk creates visualization probrems.
vol = Volume()
volNameStk = self._getPath('volume.stk')
volName = volNameStk.replace(".stk", ".vol")
pwutils.createLink(volNameStk, volName)
vol.setFileName(volName)
vol.setSamplingRate(imgSet.getSamplingRate())
self._defineOutputs(outputVolume=vol)
self._defineSourceRelation(self.inputParticles, vol)
def _convertCoords(self, micSet, tmpDir, coordsType):
""" Link specified coord set to tmpDir folder and convert it to .pos files"""
coordTypes = {
'autopick': 'coordinates.sqlite',
'rejected': 'coordinates_rejected.sqlite'
}
coordsFnIn = self.protocol._getPath(coordTypes[coordsType])
coordsFnOut = pwutils.join(tmpDir, 'coordinates.sqlite')
pwutils.createLink(coordsFnIn, coordsFnOut)
coordSet = SetOfCoordinates(filename=coordsFnOut)
coordSet.setMicrographs(micSet)
from .convert import writeSetOfCoordinatesXmipp
writeSetOfCoordinatesXmipp(tmpDir, coordSet, ismanual=False)
def _processMovie(self, movie):
fn = movie.getAttributeValue('_originalFileName', movie.getFileName())
baseName = os.path.basename(fn)
compression = self.getEnumText('compression')
pwutils.createLink(fn, self._getTmpPath(baseName))
args = "--i %s --o ../extra/ " % baseName
args += "--compression %s " % compression
# TODO: Check if deflateLevel is only valid for zip (deflate)
if compression == 'zip': # deflate
args += "--deflate_level %d" % self.deflateLevel
if self.getInputGain():
args += "--gain gain_estimate.bin "
self.runJob('relion_convert_to_tiff', args, cwd=self._getTmpPath())
def runGautomatch(micName, refStack, workDir, args, env=None):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
if refStack is not None:
args = ' %s --T %s %s' % (outMic, refStack, args)
else:
args = ' %s %s' % (outMic, args)
pwutils.runJob(None, getProgram(), args, env=env)
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def _preprocessMicrographRow(self, img, imgRow):
# Temporarly convert the few micrographs to tmp and make sure
# they are in 'mrc' format
# Get basename and replace extension by 'mrc'
newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc')
newPath = self._getExtraPath(newName)
# If the micrographs are in 'mrc' format just create a link
# if not, convert to 'mrc'
if img.getFileName().endswith('mrc'):
pwutils.createLink(img.getFileName(), newPath)
else:
self._ih.convert(img, newPath)
# The command will be launched from the working dir
# so, let's make the micrograph path relative to that
img.setFileName(os.path.join('extra', newName))
def _preprocessMicrographRow(self, img, imgRow):
# Temporarly convert the few micrographs to tmp and make sure
# they are in 'mrc' format
# Get basename and replace extension by 'mrc'
newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc')
newPath = self._getExtraPath(newName)
# If the micrographs are in 'mrc' format just create a link
# if not, convert to 'mrc'
if img.getFileName().endswith('mrc'):
pwutils.createLink(img.getFileName(), newPath)
else:
self._ih.convert(img, newPath)
# The command will be launched from the working dir
# so, let's make the micrograph path relative to that
img.setFileName(os.path.join('extra', newName))
def runGempicker(micName, workingDir, useGPU, args):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
refDir = join(workingDir, 'templates')
maskSchDir = join(workingDir, 'maskSch')
args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir, refDir,
maskSchDir)
# Run Gempicker:
for mode in [0, 1]:
pwutils.runJob(None, getProgram(useGPU), args + ' --mode=%d' % mode)
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def denoisingStep(self):
input_mics = self.inputMicrographs.get()
# Create links to the movies desired to denoise in tmp folder (subset case, janni only accepts directories
# and work with all the files contained in them)
for mic in input_mics:
micName = mic.getFileName()
createLink(mic.getFileName(), self._getTmpPath(basename(micName)))
args = "denoise {}/ {}/ {}".format(self._getTmpPath(),
self._getTmpPath(),
self.getInputModel())
Plugin.runCryolo(self, 'janni_denoise.py', args)
# Move the resulting denoised files to the extra folder
[
moveFile(file, self._getExtraPath(basename(file)))
for file in glob.glob(self._getTmpPath(join('tmp', '*.mrc')))
]
def runGempicker(micName, workingDir, useGPU, args, log=None):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
refDir = join(workingDir, 'templates')
maskSchDir = join(workingDir, 'maskSch')
args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir,
refDir, maskSchDir)
# Run Gempicker:
for mode in [0, 1]:
pwutils.runJob(log, getProgram(useGPU), args + ' --mode=%d' % mode,
env=getEnviron())
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def copyOrLinkBinary(self,
imgRow,
label,
basePath,
destBasePath,
copyFiles=False):
index, imgPath = relionToLocation(imgRow.get(label))
baseName = os.path.join(os.path.dirname(imgPath),
os.path.basename(imgPath))
os.makedirs(os.path.join(destBasePath, os.path.dirname(imgPath)),
exist_ok=True)
newName = os.path.join(destBasePath, baseName)
if not os.path.exists(newName):
if copyFiles:
pwutils.copyFile(os.path.join(basePath, imgPath), newName)
else:
pwutils.createLink(os.path.join(basePath, imgPath), newName)
imgRow.set(label, locationToRelion(index, newName))
def _getConvertFunc(self, img):
""" Return a function that will link or convert the input
file depending if the input img has a format valid for Relion.
"""
srcFile = img.getFileName()
if srcFile.endswith('mrcs'):
return lambda s, d: pwutils.createLink(s, d)
else:
return lambda s, d: self._ih.convert(s, d)
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
self.runJob(program, args)
def _createLink(self, movie):
movieFn = movie.getFileName()
if movieFn.endswith("mrcs"):
pwutils.createLink(movieFn, self._getMovieFn(movie))
def create_movies_project(request):
if request.is_ajax():
import os
from pyworkflow.object import Pointer
from pyworkflow.em.protocol import ProtImportMovies
from pyworkflow.em.packages.xmipp3 import ProtMovieAlignment
# Create a new project
projectName = request.GET.get('projectName')
# Filename to use as test data
testDataKey = request.GET.get('testData')
manager = getServiceManager('movies')
writeCustomMenu(manager.protocols)
project = manager.createProject(projectName, runsView=1,
hostsConf=manager.hosts,
protocolsConf=manager.protocols
)
project.getSettings().setLifeTime(14)
project.saveSettings()
# copyFile(customMenu, project.getPath('.config', 'protocols.conf'))
# Create symbolic link for uploads
projectPath = manager.getProjectPath(projectName)
dest = os.path.join(projectPath,'Uploads')
# @todo: this path to uploads dir should be configurable outside the code...
source = "/services/scipion/data/uploads/"+ projectName
pwutils.path.makePath(source)
pwutils.createLink(source, dest)
# 1. Import movies
if testDataKey :
attr = getAttrTestFile(testDataKey)
path_test = attr['path']
for f in os.listdir(path_test):
# Create a symbolic link for each file
file_path = os.path.join(path_test, f)
source_file = os.path.join(source, f)
pwutils.createAbsLink(file_path, source_file)
label_import = "import movies ("+ testDataKey +")"
protImport = project.newProtocol(ProtImportMovies, objLabel=label_import)
protImport.filesPath.set(attr["filesPath"])
protImport.voltage.set(attr['voltage'])
protImport.sphericalAberration.set(attr['sphericalAberration'])
protImport.amplitudeContrast.set(attr['amplitudeContrast'])
protImport.magnification.set(attr['magnification'])
protImport.samplingRate.set(attr['samplingRate'])
project.launchProtocol(protImport, wait=True)
else:
protImport = project.newProtocol(ProtImportMovies, objLabel='import movies')
project.saveProtocol(protImport)
# 2. Movie Alignment
protMovAlign = project.newProtocol(ProtMovieAlignment)
protMovAlign.setObjLabel('xmipp - movie alignment')
protMovAlign.inputMovies.set(protImport)
protMovAlign.inputMovies.setExtended('outputMovies')
project.saveProtocol(protMovAlign)
return HttpResponse(mimetype='application/javascript')
def convertInputStep(self):
inputParticles = self.inputParticles.get()
firstCoord = inputParticles.getFirstItem().getCoordinate()
self.hasMicName = firstCoord.getMicName() is not None
inputMics = self._getMicrographs()
self.alignType = inputParticles.getAlignment()
self.downFactor = self.ctfDownFactor.get()
# create a tmp set for matching mics
self.matchingMics = self._createSetOfMicrographs(suffix='_tmp')
self.matchingMics.copyInfo(inputMics)
if self.downFactor != 1.:
self.matchingMics.setDownsample(self.downFactor)
# create a tmp set for coords
coords = self._createSetOfCoordinates(inputMics, suffix='_tmp')
newCoord = Coordinate()
self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate()
if self.scale != 1.0:
print "Scaling coordinates by a factor *%0.2f*" % self.scale
# Create the micrograph dicts
micDict = {} # dict with micName or micId
micBaseDict = {} # dict with micName (just basename)
micKey2 = None
insertedMics = {}
for mic in inputMics:
if self.hasMicName:
micKey = mic.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = mic.getObjId()
if micKey in micDict:
print ">>> ERROR: micrograph key %s is duplicated!" % micKey
print " Used in micrographs:"
print " - %s" % micDict[micKey].getLocation()
print " - %s" % mic.getLocation()
raise Exception("Micrograph key %s is duplicated!" % micKey)
micDict[micKey] = mic.clone()
if self.hasMicName:
micBaseDict[micKey2] = mic.clone()
# match the mic from coord with micDict
for particle in inputParticles:
coord = particle.getCoordinate() or None
if coord is None:
print "Skipping particle, coordinates not found"
continue
if self.hasMicName:
micKey = coord.getMicName()
micKey2 = pwutils.removeBaseExt(micKey)
else:
micKey = coord.getMicId()
# find the mapping by micName (with or without ext) or micId
mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None)
if mic is None:
print "Skipping particle, key %s not found" % micKey
else:
newCoord.copyObjId(particle)
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
newCoord.setPosition(xCoor * self.scale, yCoor * self.scale)
else:
newCoord.setPosition(x * self.scale, y * self.scale)
newCoord.setMicrograph(mic)
coords.append(newCoord)
if mic.getObjId() not in insertedMics:
insertedMics[mic.getObjId()] = mic
self.matchingMics.append(mic)
ih = em.ImageHandler()
# We convert matching micrographs if they are not *.mrc
for mic in self.matchingMics:
# Create micrograph dir
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if self.downFactor != 1.:
ih.scaleFourier(micName, outMic, self.downFactor)
sps = inputMics.getScannedPixelSize() * self.downFactor
self._params['scannedPixelSize'] = sps
else:
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(micName, outMic)
# Write out coordinate files and sets
writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics)
coords.clear()
pwutils.cleanPath(coords.getFileName())
self.matchingMics.write()
self.matchingMics.close()