def getBoxSize(self, coordFile):
""" Try to infer the box size from the given coordinate file.
In the case of .box files, the size is the 3rd column
In the case of .json files, we will look for file
e2boxercache/base.json
"""
if coordFile.endswith('.box'):
md = MetaData()
md.readPlain(coordFile, "xcoor ycoor particleSize")
return md.getValue(MDL_PICKING_PARTICLE_SIZE, md.firstObject())
elif coordFile.endswith('.json'):
infoDir = os.path.dirname(coordFile)
# Still go one level up of info dir
jsonBase = pwutils.join(os.path.dirname(infoDir), 'e2boxercache',
'base.json')
jsonBase2 = pwutils.join(infoDir, 'project.json')
if pwutils.exists(jsonBase):
jsonDict = loadJson(jsonBase)
if 'box_size' in jsonDict:
return int(jsonDict["box_size"])
elif pwutils.exists(jsonBase2):
jsonDict = loadJson(jsonBase2)
if 'global.boxsize' in jsonDict:
return int(jsonDict["global.boxsize"])
return None
def readSetOfCoordinates(outputDir, micSet, coordSet):
""" Read from Bsoft .star files.
Params:
outputDir: the directory where the .star files are.
micSet: the SetOfMicrographs to associate the .star, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
boxSize = 100
for mic in micSet:
outputFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'star'))
if exists(outputFile):
posMd = md.MetaData(outputFile)
for objId in posMd:
coordRow = rowFromMd(posMd, objId)
coord = rowToCoordinate(coordRow)
boxSize = 2 * float(coordRow.getValue("particle.x_origin", 50))
coord.setMicrograph(mic)
coord.setX(coord.getX())
coord.setY(coord.getY())
coordSet.append(coord)
# Add an unique ID that will be propagated to particles
# posMd.setValue(md.MDL_PARTICLE_ID, long(coord.getObjId()), objId)
# reading origin.x value and converting to particle
# size, can change, we take last value
coordSet.setBoxSize(boxSize)
def createOutputStep(self):
# Check that the indexing created proper output
assert (pwutils.exists(self.getOutputReflFile()))
assert (pwutils.exists(self.getOutputModelFile()))
outputSet = self._createSetOfIndexedSpots()
outputSet.setDialsModel(self.getOutputModelFile())
outputSet.setDialsRefl(self.getOutputReflFile())
try:
outputSet.setDialsHtml(self.getOutputHtmlFile())
except FileNotFoundError:
self.info(self.getError())
outputSet.write()
self._defineOutputs(outputScaledSpots=outputSet)
def _openHTML(self, e=None):
reportHtml = self.protocol.createHtmlReport()
reportPath = reportHtml.reportPath
if pwutils.exists(reportPath):
text._open_cmd(reportPath)
else:
self.showInfo('Your html file is not ready yet. Please try again in a minute.')
def createMetadataImageStep(self):
mdUntilted = md.MetaData()
mdTilted = md.MetaData()
# for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = pwutils.removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in these micrographs
if pwutils.exists(fnMicU):
mdMicU = md.MetaData(fnMicU)
mdPosU = md.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = pwutils.removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = md.MetaData(fnMicT)
mdPosT = md.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadata (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
def createMetadataImageStep(self):
mdUntilted = md.MetaData()
mdTilted = md.MetaData()
# for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = pwutils.removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in these micrographs
if pwutils.exists(fnMicU):
mdMicU = md.MetaData(fnMicU)
mdPosU = md.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = pwutils.removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = md.MetaData(fnMicT)
mdPosT = md.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadata (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
def readCTFModel(ctfModel, filename):
jsonDict = loadJson(filename)
keyPos = None
ctfPhaseShift = 0.0
if jsonDict.has_key('ctf_frame'):
keyPos = jsonDict['ctf_frame'][1]
elif jsonDict.has_key('ctf'):
keyPos = jsonDict['ctf'][0]
else:
setWrongDefocus(ctfModel)
if keyPos:
defocus = float(keyPos['defocus'])
defocusAngle = float(keyPos['dfang'])
dfdiff = float(keyPos['dfdiff'])
ampcont = float(keyPos['ampcont'])
defocusU = 10000.0 * defocus + 5000.0 * dfdiff
defocusV = 20000.0 * defocus - defocusU
ctfPhaseShift = calculatePhaseShift(ampcont)
ctfModel.setStandardDefocus(defocusU, defocusV, defocusAngle)
if jsonDict.has_key('ctf_im2d'):
# psdFile = jsonDict['ctf_im2d']['__image__'][0]
fnBase = pwutils.removeExt(filename) + '_jsonimg'
psdFile = "1@%s.hdf" % fnBase
if pwutils.exists(psdFile):
ctfModel.setPsdFile(psdFile)
ctfModel.setPhaseShift(float(ctfPhaseShift))
def __getRemoteFile(self, sourceFilePath, targetFilePath, gatewayHosts,
sftp):
"""
Send local file to remote machine
sourceFilePath -- Source file path (/file path/...).
targetFilePath -- Target file path (/file path/...).
sftp -- sftp connection.
"""
log.info("Getting " + sourceFilePath + " to " + targetFilePath)
# Check if file already existsFilePath and it is up to date
existsFilePath = False
if exists(targetFilePath):
if self.__getRemoteSHA1(
sourceFilePath,
self.ssh) == self.__getLocalSHA1(targetFilePath):
existsFilePath = True
log.info(targetFilePath + " already existed")
if not existsFilePath:
makeFilePath(targetFilePath)
try:
sftp.get(sourceFilePath, targetFilePath)
except IOError as err:
log.error("Fail getting remote file " + sourceFilePath +
" to local file " + targetFilePath + " - " +
str(err))
raise
def _getValues(self):
fileName = self.protocol._getFileName('outputAngles')
planethres = self.planethres.get()
r = []
theta = []
datap = []
zaxis = []
if pwutils.exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("particletilt_list"):
tiltpairs = jsonPosDict["particletilt_list"]
maxcolorval = max(tiltpairs, key=lambda x: x[3])[3]
for tp in tiltpairs:
if tp[3] > planethres:
continue
datap.append(tp[0])
r.append(tp[1])
theta.append(math.radians(tp[2]))
# Color the Z axis out of planeness
zaxis.append(self._computeRGBcolor(tp[3], 0, maxcolorval))
return datap, r, theta, zaxis
def _visualize(self, obj, **args):
if hasattr(obj, 'outputVolumes'):
XmippViewer._visualize(self,self.protocol.outputVolumes)
fnBasis=self.protocol._getExtraPath('split_pc1.vol')
if exists(fnBasis):
self._views.append(DataView(fnBasis))
return self._views
def _openHTML(self, e=None):
reportHtml = self.protocol.createHtmlReport()
reportPath = reportHtml.reportPath
if pwutils.exists(reportPath):
text._open_cmd(reportPath)
else:
self.showInfo(
'Your html file is not ready yet. Please try again in a minute.'
)
def _summary(self):
summary = []
fname = os.path.abspath(
join(self._getPath('emxData'),
self.outputPrefix.get() + '.emx'))
if exists(fname):
summary.append('Exported %s to %s' %
(self._exportTypes[self.inputType.get()], fname))
return summary
def _showHtmlReport(self, paramName=None):
reportPath = self.protocol._getFileName('reportHtml',
run=self.protocol._getRun())
if pwutils.exists(reportPath):
text._open_cmd(reportPath, self.getTkRoot())
else:
self.showInfo(
'Your html report is not ready yet. Please try again in a minute.'
)
def _getHtml(self):
try:
reportPath = self.protocol.getOutputHtmlFile()
except AttributeError:
return None
if pwutils.exists(reportPath):
return reportPath
else:
return None
def _summary(self):
summary = []
if exists(self._getExtraPath('histogram.png')):
results = self._parseOutput()
summary.append('Mean resolution: %0.2f A' % results[0])
summary.append('Median resolution: %0.2f A' % results[1])
else:
summary.append("Output is not ready yet.")
return summary
def _summary(self):
summary = []
if exists(self._getExtraPath('histogram.png')):
results = self._parseOutput()
summary.append('Mean resolution: %0.2f A' % results[0])
summary.append('Median resolution: %0.2f A' % results[1])
else:
summary.append("Output is not ready yet.")
return summary
def _summary(self):
""" Should be overwritten in subclasses to
return summary message for NORMAL EXECUTION.
"""
summary = []
postStarFn = self._getExtraPath("postprocess.star")
if exists(postStarFn):
mdResol = md.RowMetaData(postStarFn)
resol = mdResol.getValue(md.RLN_POSTPROCESS_FINAL_RESOLUTION)
summary.append("Final resolution: *%0.2f A*" % resol)
return summary
def _summary(self):
""" Should be overwritten in subclasses to
return summary message for NORMAL EXECUTION.
"""
summary = []
postStarFn = self._getExtraPath("postprocess.star")
if exists(postStarFn):
mdResol = md.RowMetaData(postStarFn)
resol = mdResol.getValue(md.RLN_POSTPROCESS_FINAL_RESOLUTION)
summary.append("Final resolution: *%0.2f A*" % resol)
return summary
def readSetOfParticles(lstFile, partSet, copyOrLink, direc):
for index, fn in iterLstFile(lstFile):
item = Particle()
# set full path to particles stack file
abspath = os.path.abspath(lstFile)
fn = abspath.replace('sets/%s' % os.path.basename(lstFile), '') + fn
newFn = pwutils.join(direc, os.path.basename(fn))
if not pwutils.exists(newFn):
copyOrLink(fn, newFn)
item.setLocation(index, newFn)
partSet.append(item)
def readCoordinates(mic, fileName, coordsSet):
if exists(fileName):
md = emlib.MetaData()
md.readPlain(fileName, 'xcoor ycoor')
for objId in md:
x = md.getValue(emlib.MDL_XCOOR, objId)
y = md.getValue(emlib.MDL_YCOOR, objId)
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def _summary(self):
summary = []
self._createFilenameTemplates()
if exists(self._getFileName('outResmapVol')):
results = self._parseOutput()
summary.append('Mean resolution: %0.2f A' % results[0])
summary.append('Median resolution: %0.2f A' % results[1])
else:
summary.append("Output is not ready yet.")
return summary
def testStepsNoGPU(self):
xmipp3Protocols = Domain.importFromPlugin('xmipp3.protocols',
doRaise=True)
protXmippPreproc = self.newProtocol(xmipp3Protocols.XmippProtPreprocessParticles,
doNormalize=True, doRemoveDust=True)
protXmippPreproc.inputParticles.set(self.protImport.outputParticles)
protXmippPreproc.setObjLabel("Xmipp preprocess steps")
protXmippPreproc._useQueue.set(True)
QUEUE_PARAMS[1]['QUEUE_FOR_JOBS'] = 'Y'
protXmippPreproc._queueParams.set(json.dumps(QUEUE_PARAMS))
# Launch protocol but wait until it finishes
self.launchProtocol(protXmippPreproc, wait=True)
# Check that job files have been created
jobFilesPath = join(pwutils.getParentFolder(protXmippPreproc.getLogPaths()[0]),
str(protXmippPreproc.getObjId()))
self.assertTrue(pwutils.exists(jobFilesPath + "-0-1.out") and
pwutils.exists(jobFilesPath + "-0-1.err") and
pwutils.exists(jobFilesPath + "-0-1.job") and
pwutils.exists(jobFilesPath + "-0-2.out") and
pwutils.exists(jobFilesPath + "-0-2.err") and
pwutils.exists(jobFilesPath + "-0-2.job"),
"Job queue files not found on log folder, job did "
"not make it to the queue.")
# Check that results have been produced
self.assertIsNotNone(protXmippPreproc.outputParticles,
"There was a problem with Xmipp preprocess particles.")
def _validate(self):
errors = []
# Check that the program exists
if not pwutils.exists(self._getProgram()):
errors.append("Binary '%s' does not exits.\n"
"Check configuration file: \n"
"~/.config/scipion/scipion.conf\n"
"and set GCTF variables properly." %
self._getProgram())
if self.useInputCtf and not self.ctfRelations.get():
errors.append("Please provide input CTFs for refinement.")
return errors
def getProgram(cls, program):
""" Return the program binary that will be used. """
if program == CTFFIND4_BIN:
# if CTFFIND4_HOME is found, use it
path = cls.getVar(CTFFIND4_HOME)
if pwutils.exists(path):
binary = os.path.join(path, 'bin', program)
else:
binary = os.path.join(cls.getHome(), program)
else:
binary = os.path.join(cls.getHome(), program)
return binary
def _validate(self):
errors = []
# Check that the program exists
if not pwutils.exists(self._getProgram()):
errors.append("Binary '%s' does not exits.\n"
"Check configuration file: \n"
"~/.config/scipion/scipion.conf\n"
"and set GCTF variables properly."
% self._getProgram())
if self.useInputCtf and not self.ctfRelations.get():
errors.append("Please provide input CTFs for refinement.")
return errors
def _showPlot(self, paramName=None):
views = []
color = self.colozaxis
rmax = self.radcut.get()
if self.displayPlot == TILT_SCATTER:
views.append(self._createScatterPlot(rmax, colorzaxis=color))
elif self.displayPlot == TILT_CONTOUR:
plotFn = self.protocol._getFileName('outputContourPlot')
if pwutils.exists(plotFn):
views.append(DataView(plotFn))
else:
raise Exception("Contour plot file not found: %s" % plotFn)
return views
def __copyLocalFile(self, sourceFilePath, targetFilePath):
"""
Send local file to remote machine
sourceFilePath -- Source file path (/file path/...).
targetFilePath -- Target file path (/file path/...).
"""
log.info("Copying " + sourceFilePath + " to " + targetFilePath)
# Check if file already existsFilePath and it is up to date
existsFilePath = False
if exists(targetFilePath):
if self.__getLocalSHA1(sourceFilePath) == self.__getLocalSHA1(
targetFilePath):
existsFilePath = True
log.info(targetFilePath + " already existed")
if not existsFilePath:
makeFilePath(targetFilePath)
shutil.copy2(sourceFilePath, targetFilePath)
def readCoordinates(mic, fn, coordsSet):
""" Parse coords file and populate coordsSet.
:param mic: input micrograph object
:param fn: input file to parse
:param coordsSet: output set of coords
"""
if exists(fn):
with open(fn, 'r') as f:
for line in f:
values = line.strip().split()
# plt coords are in Imagic style
x = float(values[1])
y = float(mic.getYDim() - float(values[0]))
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def readCoordinates(mic, fileName, coordsSet, invertY=False):
if pwutils.exists(fileName):
jsonPosDict = loadJson(fileName)
if jsonPosDict.has_key("boxes"):
boxes = jsonPosDict["boxes"]
for box in boxes:
x, y = box[:2]
if invertY:
y = mic.getYDim() - y
coord = Coordinate()
coord.setPosition(x, y)
coord.setMicrograph(mic)
coordsSet.append(coord)
def importCoordinates3D(self, fileName, addCoordinate):
from tomo.objects import Coordinate3D
if pwutils.exists(fileName):
ext = pwutils.getExt(fileName)
if ext == ".txt":
md = MetaData()
md.readPlain(fileName, "xcoor ycoor zcoor")
for objId in md:
x = md.getValue(MDL_XCOOR, objId)
y = md.getValue(MDL_YCOOR, objId)
z = md.getValue(MDL_ZCOOR, objId)
coord = Coordinate3D()
addCoordinate(coord, x, y, z)
else:
raise Exception(
'Unknown extension "%s" to import Eman coordinates' % ext)
def _getIterClasses(self, it, clean=False):
""" Return a classes .sqlite file for this iteration.
If the file doesn't exists, it will be created by
converting from this iteration mltomo_it??????_img.xmd file.
"""
data_classes = self._getFileName('classes_scipion', iter=it)
if clean:
pwutils.cleanPath(data_classes)
if not pwutils.exists(data_classes):
clsSet = SetOfClassesVol(filename=data_classes)
clsSet.setImages(self.inputVols.get())
self._fillClassesFromIter(clsSet, it)
clsSet.write()
clsSet.close()
return data_classes
def _validate(self):
""" Should be overwritten in subclasses to
return summary message for NORMAL EXECUTION.
"""
errors = []
mtfFile = self.mtf.get()
if mtfFile and not exists(mtfFile):
errors.append("Missing MTF-file '%s'" % mtfFile)
protClassName = self.protRefine.get().getClassName()
if protClassName.startswith('SpiderProtRefinement'):
errors.append("Relion local resolution protocol is not "
"implemented for Spider - refinement.")
if protClassName.startswith('XmippProtReconstructHighRes'):
errors.append("Relion local resolution protocol is not "
"implemented for Xmipp - highres.")
return errors
def _validate(self):
""" Should be overwritten in subclasses to
return summary message for NORMAL EXECUTION.
"""
errors = []
mtfFile = self.mtf.get()
if mtfFile and not exists(mtfFile):
errors.append("Missing MTF-file '%s'" % mtfFile)
protClassName = self.protRefine.get().getClassName()
if protClassName.startswith('SpiderProtRefinement'):
errors.append("Relion local resolution protocol is not "
"implemented for Spider - refinement.")
if protClassName.startswith('XmippProtReconstructHighRes'):
errors.append("Relion local resolution protocol is not "
"implemented for Xmipp - highres.")
return errors
def importAngles(self, fileName, addAngles):
if pwutils.exists(fileName):
ext = pwutils.getExt(fileName)
if ext == ".json":
fnBase = pwutils.replaceBaseExt(fileName, 'hdf')
keyName = 'tiltparams_micrographs/' + fnBase.replace(
'_info', '')
jsonAngDict = loadJson(fileName)
if keyName in jsonAngDict:
angles = jsonAngDict[keyName]
tilt, y2, y = angles[:3] # y2=tilted, y=gamma(untilted)
ang = Angles()
# TODO: check this conversion
ang.setAngles(y, y2, tilt)
addAngles(ang)
else:
raise Exception(
'Unknown extension "%s" to import Eman tilt pair angles' %
ext)
def importCoordinates3D(self, fileName, addCoordinate):
from tomo.objects import Coordinate3D
if pwutils.exists(fileName):
ext = pwutils.getExt(fileName)
if ext == ".json":
jsonPosDict = loadJson(fileName)
boxes = []
if "boxes_3d" in jsonPosDict:
boxes = jsonPosDict["boxes_3d"]
if boxes:
for box in boxes:
x, y, z = box[:3]
coord = Coordinate3D()
addCoordinate(coord, x, y, z)
else:
raise Exception(
'Unknown extension "%s" to import Eman coordinates' % ext)
def test_movies(self):
print(magentaStr("\n==> Importing data - movies:"))
protImport = self.runImportMovies()
outputMovies = getattr(protImport, 'outputMovies', None)
self.assertIsNotNone(outputMovies)
inputMovies = protImport.outputMovies
print(magentaStr("\n==> Testing cistem - unblur:"))
prot = self.newProtocol(CistemProtUnblur, alignFrame0=2, alignFrameN=6)
prot.inputMovies.set(inputMovies)
self.launchProtocol(prot)
outputMics = getattr(prot, 'outputMicrographsDoseWeighted', None)
self.assertIsNotNone(outputMics)
self.assertEqual(protImport.outputMovies.getSize(),
outputMics.getSize())
for mic in outputMics:
micFn = mic.getFileName()
self.assertTrue(exists(self.proj.getPath(micFn)))
def createCtfModelStep(self):
inputSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(inputSet)
for particle in inputSet:
coord = particle.getCoordinate()
if coord is None:
continue
x, y = coord.getPosition()
if self.applyShifts:
shifts = getShifts(particle.getTransform(), self.alignType)
xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1])
xNew, yNew = (xCoor * self.scale, yCoor * self.scale)
else:
xNew, yNew = (x * self.scale, y * self.scale)
micBase = pwutils.removeBaseExt(coord.getMicName())
for key in self.matchingMics:
micKey = pwutils.removeBaseExt(key.getFileName())
if micBase in micKey:
# micName from mic and micName from coord may be different
ctfFn = pwutils.join(self._getExtraPath(micKey),
micKey + '_local.star')
if pwutils.exists(ctfFn):
mdFn = md.MetaData(ctfFn)
for row in md.iterRows(mdFn):
coordX = row.getValue(md.RLN_IMAGE_COORD_X)
coordY = row.getValue(md.RLN_IMAGE_COORD_Y)
if (int(xNew), int(yNew)) == (coordX, coordY):
newPart = particle.clone()
rowToCtfModel(row, newPart.getCTF())
partSet.append(newPart)
self._defineOutputs(outputParticles=partSet)
self._defineTransformRelation(inputSet, partSet)
