def importTomogramsStep(self, pattern, samplingRate):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
# Create a Volume template object
tomo = Tomogram()
tomo.setSamplingRate(samplingRate)
imgh = ImageHandler()
tomoSet = self._createSetOfTomograms()
tomoSet.setSamplingRate(samplingRate)
self._parseAcquisitionData()
for fileName, fileId in self.iterFiles():
x, y, z, n = imgh.getDimensions(fileName)
if fileName.endswith('.mrc') or fileName.endswith('.map'):
fileName += ':mrc'
if z == 1 and n != 1:
zDim = n
n = 1
else:
zDim = z
else:
zDim = z
origin = Transform()
if self.setOrigCoord.get():
origin.setShiftsTuple(self._getOrigCoord())
else:
origin.setShifts(x / -2. * samplingRate,
y / -2. * samplingRate,
zDim / -2. * samplingRate)
tomo.setOrigin(origin) # read origin from form
newFileName = _getUniqueFileName(self.getPattern(),
fileName.split(':')[0])
tsId = removeExt(newFileName)
tomo.setTsId(tsId)
if fileName.endswith(':mrc'):
fileName = fileName[:-4]
createAbsLink(fileName, abspath(self._getExtraPath(newFileName)))
tomo.setAcquisition(self._extractAcquisitionParameters(fileName))
if n == 1: # One volume per file
tomo.cleanObjId()
tomo.setFileName(self._getExtraPath(newFileName))
tomoSet.append(tomo)
else: # A stack of volumes per file (not common)
for index in range(1, n + 1):
tomo.cleanObjId()
tomo.setLocation(index, self._getExtraPath(newFileName))
tomoSet.append(tomo)
self._defineOutputs(outputTomograms=tomoSet)
def applyAlignmentStep(self, tsObjId):
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
path.makePath(extraPrefix)
outputTsFileName = os.path.join(extraPrefix, "%s.mrc" % tsId)
ts.applyTransform(outputTsFileName)
outputInterpolatedSetOfTiltSeries = self.getOutputInterpolatedSetOfTiltSeries()
newTs = tomoObj.TiltSeries(tsId=tsId)
newTs.copyInfo(ts)
outputInterpolatedSetOfTiltSeries.append(newTs)
for index, tiltImage in enumerate(ts):
newTi = tomoObj.TiltImage()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(index + 1, outputTsFileName)
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
newTs.write()
outputInterpolatedSetOfTiltSeries.update(newTs)
outputInterpolatedSetOfTiltSeries.updateDim()
outputInterpolatedSetOfTiltSeries.write()
self._store()
def test_tomoProjectionOutputTiltSeriesDimension(self):
ih = ImageHandler()
outputDimensions = ih.getDimensions(
self.protTomoProjection.outputProjectedSetOfTiltSeries.
getFirstItem().getFirstItem().getFileName())
self.assertTrue(outputDimensions == (256, 256, 61, 1))
def importMaskStep(self, path, samplingRate):
""" Copy mask from maskPath.
"""
self.info("Using mask path: '%s'" % path)
# Copy the image file into the project
dst = self._getExtraPath(basename(path))
dst, cleaned = cleanFileName(dst)
pwutils.copyFile(path, dst)
# Retrieve image dimensions
imgh = ImageHandler()
_, _, z, n = imgh.getDimensions(dst)
# Create a 2D or 3D Mask, consider the case of n>1
# as the case of some volume maps in mrc format
if z > 1 or n > 1:
mask = VolumeMask()
else:
mask = Mask()
mask.setFileName(dst)
mask.setSamplingRate(samplingRate)
self._defineOutputs(**{self._possibleOutputs.outputMask.name: mask})
def getTomoSetFromStar(prot, starFile):
samplingRate = prot.pixelSize.get()
imgh = ImageHandler()
tomoTable = Table()
tomoTable.read(starFile)
tomoList = [row.get(TOMO_NAME, FILE_NOT_FOUND) for row in tomoTable]
prot.tomoList = List(tomoList)
tomoNamesUnique = list(set(tomoList))
# Create a Volume template object
tomo = Tomogram()
tomo.setSamplingRate(samplingRate)
for fileName in tomoNamesUnique:
x, y, z, n = imgh.getDimensions(fileName)
if fileName.endswith('.mrc') or fileName.endswith('.map'):
if z == 1 and n != 1:
zDim = n
n = 1
else:
zDim = z
else:
zDim = z
origin = Transform()
origin.setShifts(x / -2. * samplingRate,
y / -2. * samplingRate,
zDim / -2. * samplingRate)
tomo.setOrigin(origin) # read origin from form
for index in range(1, n + 1):
tomo.cleanObjId()
tomo.setLocation(index, fileName)
tomo.setAcquisition(TomoAcquisition(**prot.acquisitionParams))
prot.tomoSet.append(tomo)
def main():
# Get arguments.
args = get_parser().parse_args()
setObj = EMSet(filename=args.setFile)
firstItem = setObj.getFirstItem()
root = pwutils.findRootFrom(args.setFile, firstItem.getFileName())
print("ROOT: ", root)
if args.output_text:
with open(args.output, 'w') as f:
for movie in setObj:
f.write('%s\n' % movie.getFileName())
elif args.print:
for item in setObj:
print(item.getFileName())
elif args.copy:
for item in setObj:
fn = os.path.join(root, item.getFileName())
print('Copying %s to %s' % (fn, args.output))
pwutils.copyFile(fn, args.output)
elif args.check_dims:
from pwem.emlib.image import ImageHandler
ih = ImageHandler()
counter = {}
for item in setObj:
fn = os.path.join(root, item.getFileName())
dims = ih.getDimensions(fn)
counter[dims] = 1 if dims not in counter else counter[dims] + 1
print('%s: %s' % (os.path.basename(fn), dims))
pwutils.prettyDict(counter)
def maskVolume(self, volFn, maskFn, index, halfString):
ih = ImageHandler()
# Check if there is a mask if not then use a spherical mask
if maskFn is None:
# Compute sphere radius and make a spherical mask
maskFn = self._getFileName('mask', index + 1)
volSize = ih.getDimensions(volFn)[0]
radius = volSize / 2 - 1
emlib.createEmptyFile(maskFn, volSize, volSize, volSize)
program = "xmipp_transform_mask"
args = "-i %s --mask circular %d --create_mask %s " % (
maskFn, -1 * radius, maskFn)
self.runJob(program, args)
else:
putils.copyFile(maskFn, self._getFileName('mask', index + 1))
maskFn = self._getFileName('mask', index + 1)
# Read the volume if it is provided
if volFn.endswith(".mrc"):
volFn = volFn + ":mrc"
# Apply mask to the volume
volMasked = self._getFileName('volume', 'masked', index + 1,
halfString)
program = 'xmipp_image_operate'
args = '-i %s --mult %s -o %s' % (volFn, maskFn, volMasked)
self.runJob(program, args)
def createOutputStep(self, tsObjId):
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
outputSetOfTomograms = self.getOutputSetOfTomograms()
newTomogram = Tomogram()
newTomogram.setLocation(os.path.join(extraPrefix, ts.getFirstItem().parseFileName(extension=".mrc")))
# Set tomogram origin
ih = ImageHandler()
xDim, yDim, zDim, _ = \
ih.getDimensions(os.path.join(extraPrefix, ts.getFirstItem().parseFileName(extension=".mrc")))
origin = Transform()
sr = self.inputSetOfTiltSeries.get().getSamplingRate()
origin.setShifts(xDim / -2. * sr,
yDim / -2. * sr,
zDim / -2 * sr)
newTomogram.setOrigin(origin)
# Set tomogram acquisition
acquisition = TomoAcquisition()
acquisition.setAngleMin(ts.getFirstItem().getTiltAngle())
acquisition.setAngleMax(ts[ts.getSize()].getTiltAngle())
acquisition.setStep(self.getAngleStepFromSeries(ts))
newTomogram.setAcquisition(acquisition)
outputSetOfTomograms.append(newTomogram)
outputSetOfTomograms.update(newTomogram)
outputSetOfTomograms.write()
self._store()
def importSubTomogramsStep(self, pattern, samplingRate):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
# Create a Volume template object
subtomo = SubTomogram()
subtomo.setSamplingRate(samplingRate)
imgh = ImageHandler()
self.subtomoSet = self._createSetOfSubTomograms()
self.subtomoSet.setSamplingRate(samplingRate)
# if self.importCoordinates.get():
# self.coords = []
# for coord3D in self.importCoordinates.get().iterCoordinates():
# self.coords.append(coord3D.clone())
# self.subtomoSet.setCoordinates3D(self.importCoordinates)
self._parseAcquisitionData()
for fileName, fileId in self.iterFiles():
x, y, z, n = imgh.getDimensions(fileName)
if fileName.endswith('.map'):
fileName += ':mrc'
if fileName.endswith('.mrc') or fileName.endswith(':mrc'):
if z == 1 and n != 1:
zDim = n
n = 1
else:
zDim = z
else:
zDim = z
origin = Transform()
origin.setShifts(x / -2. * samplingRate, y / -2. * samplingRate,
zDim / -2. * samplingRate)
subtomo.setOrigin(origin) # read origin from form
newFileName = _getUniqueFileName(self.getPattern(), fileName)
# newFileName = abspath(self._getVolumeFileName(fileName))
if fileName.endswith(':mrc'):
fileName = fileName[:-4]
createAbsLink(fileName, self._getExtraPath(newFileName))
if n == 1:
self._addSubtomogram(subtomo, self._getExtraPath(fileName),
self._getExtraPath(newFileName))
else:
for index in range(1, n + 1):
self._addSubtomogram(subtomo,
self._getExtraPath(fileName),
self._getExtraPath(newFileName),
index=index)
def assignTransformationMatricesStep(self):
self.getOutputAssignedTransformSetOfTiltSeries()
inputSetOfTiltSeries = self.inputSetOfTiltSeries.get()
for ts in inputSetOfTiltSeries:
tsId = ts.getTsId()
tsFileName = ts.getFirstItem().parseFileName(extension='')
for tmFilePath, _ in self.iterFiles():
tmFileName = os.path.basename(os.path.splitext(tmFilePath)[0])
if tsFileName == tmFileName:
alignmentMatrix = utils.formatTransformationMatrix(
tmFilePath)
newTs = tomoObj.TiltSeries(tsId=tsId)
newTs.copyInfo(ts)
self.outputAssignedTransformSetOfTiltSeries.append(newTs)
for index, tiltImage in enumerate(ts):
newTi = tomoObj.TiltImage()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(tiltImage.getLocation())
transform = data.Transform()
if tiltImage.hasTransform():
previousTransform = tiltImage.getTransform(
).getMatrix()
newTransform = alignmentMatrix[:, :, index]
previousTransformArray = np.array(
previousTransform)
newTransformArray = np.array(newTransform)
outputTransformMatrix = np.matmul(
previousTransformArray, newTransformArray)
transform.setMatrix(outputTransformMatrix)
newTi.setTransform(transform)
else:
transform.setMatrix(alignmentMatrix[:, :, index])
newTi.setTransform(transform)
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(
newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
newTs.write(properties=False)
self.outputAssignedTransformSetOfTiltSeries.update(newTs)
self.outputAssignedTransformSetOfTiltSeries.updateDim()
self.outputAssignedTransformSetOfTiltSeries.write()
self._store()
def resizeStep(self, tomoMask):
ih = ImageHandler()
fileName = tomoMask.getFileName()
x, y, z, _ = ih.getDimensions(tomoMask)
nx, ny, nz, _ = ih.getDimensions(self.inTomos.get().getFirstItem())
with mrcfile.open(fileName, permissive=True) as mrc:
originalMask = np.round_(mrc.data)
rx = nx / x
ry = ny / y
rz = nz / z
resizedMask = zoom(originalMask, (rx, ry, rz), order=0)
# Save resized data into a mrc file
resizedFileName = self._getResizedMaskFileName(fileName)
with mrcfile.open(resizedFileName, mode='w+') as mrc:
mrc.set_data(np.round_(resizedMask))
self.resizedFileList.append(resizedFileName)
def _relionTomoStar2Subtomograms(prot, outputSubTomogramsSet, tomoTable, invert):
ih = ImageHandler()
samplingRate = outputSubTomogramsSet.getSamplingRate()
for row, inSubtomo in zip(tomoTable, prot.inputSubtomos.get()):
subtomo = SubTomogram()
coordinate3d = Coordinate3D()
transform = Transform()
origin = Transform()
volname = row.get(TOMO_NAME, FILE_NOT_FOUND)
subtomoFn = row.get(SUBTOMO_NAME, FILE_NOT_FOUND)
subtomo.setVolName(managePath4Sqlite(volname))
subtomo.setTransform(transform)
subtomo.setAcquisition(TomoAcquisition())
subtomo.setClassId(row.get('rlnClassNumber', 0))
subtomo.setSamplingRate(samplingRate)
subtomo.setCoordinate3D(coordinate3d) # Needed to get later the tomogram pointer via getCoordinate3D()
x = row.get(COORD_X, 0)
y = row.get(COORD_Y, 0)
z = row.get(COORD_Z, 0)
tiltPrior = row.get(TILT_PRIOR, 0)
psiPrior = row.get(PSI_PRIOR, 0)
# ctf3d = row.get(CTF_MISSING_WEDGE, FILE_NOT_FOUND)
coordinate3d = subtomo.getCoordinate3D()
coordinate3d.setX(float(x), BOTTOM_LEFT_CORNER)
coordinate3d.setY(float(y), BOTTOM_LEFT_CORNER)
coordinate3d.setZ(float(z), BOTTOM_LEFT_CORNER)
coordinate3d._3dcftMrcFile = inSubtomo.getCoordinate3D()._3dcftMrcFile # Used for the ctf3d in Relion 3.0 (tomo)
M = _getTransformMatrix(row, invert)
transform.setMatrix(M)
subtomo.setCoordinate3D(coordinate3d)
subtomo._tiltPriorAngle = Float(tiltPrior)
subtomo._psiPriorAngle = Float(psiPrior)
# Set the origin and the dimensions of the current subtomogram
x, y, z, n = ih.getDimensions(subtomoFn)
zDim, _ = manageIhDims(subtomoFn, z, n)
origin.setShifts(x / -2. * samplingRate, y / -2. * samplingRate, zDim / -2. * samplingRate)
subtomo.setOrigin(origin)
subtomo.setFileName(managePath4Sqlite(subtomoFn))
# if subtomo is in a vesicle
if 'tid_' in subtomoFn:
vesicleId = subtomoFn.split('tid_')[1]
vesicleId = vesicleId[0]
scoor = subtomo.getCoordinate3D()
scoor.setGroupId(vesicleId)
subtomo.setCoordinate3D(scoor)
# Add current subtomogram to the output set
outputSubTomogramsSet.append(subtomo)
def _prepareRecalCommand(self, ctfModel):
if self.recalculate:
self._defineRecalValues(ctfModel)
self._createFilenameTemplates()
self._program = 'xmipp_ctf_estimate_from_psd_fast'
self._args = "--psd %(psdFn)s "
line = ctfModel.getObjComment().split()
# get the size and the image of psd
imgPsd = ctfModel.getPsdFile()
psdFile = os.path.basename(imgPsd)
imgh = ImageHandler()
size, _, _, _ = imgh.getDimensions(imgPsd)
mic = ctfModel.getMicrograph()
micDir = self._getMicrographDir(mic)
downFactor = self._calculateDownsampleList(
mic.getSamplingRate())[0]
params = dict(self.getCtfParamsDict())
params.update(self.getRecalCtfParamsDict())
# FIXME Where does this variable come from
params.update({'psdFn': fnPsd, 'defocusU': float(line[0])})
# Mapping between base protocol parameters and the package specific
# command options
self.__params = {
'sampling_rate': params['samplingRate'],
'downSamplingPerformed': downFactor,
'kV': params['voltage'],
'Cs': params['sphericalAberration'],
'min_freq': line[3],
'max_freq': line[4],
'defocusU': params['defocusU'],
'Q0': params['ampContrast'],
'defocus_range': 5000,
'ctfmodelSize': size
}
if self.findPhaseShift:
fnCTFparam = self._getFileName('ctfParam',
micBase=self._getMicBase(mic))
mdCTFParam = md.MetaData(fnCTFparam)
phase_shift = mdCTFParam.getValue(md.MDL_CTF_PHASE_SHIFT,
mdCTFParam.firstObject())
self.__params['VPP_radius'] = 0.005
self.__params['phase_shift'] = phase_shift
for par, val in self.__params.items():
self._args += " --%s %s" % (par, str(val))
def generateOutputStackStep(self, tomoObjId):
tomo = self.inputSetOfTomograms.get()[tomoObjId]
tomoId = os.path.splitext(os.path.basename(tomo.getFileName()))[0]
extraPrefix = self._getExtraPath(tomoId)
outputProjectedSetOfTiltSeries = self.getOutputProjectedSetOfTiltSeries(
)
newTs = tomoObj.TiltSeries(tsId=tomoId)
newTs.copyInfo(tomo)
newTs.setTsId(tomoId)
# Add origin to output tilt-series
origin = Transform()
outputProjectedSetOfTiltSeries.append(newTs)
tiltAngleList = self.getTiltAngleList()
for index in range(self.getProjectionRange()):
newTi = tomoObj.TiltImage()
newTi.setTiltAngle(tiltAngleList[index])
newTi.setTsId(tomoId)
newTi.setLocation(
index + 1,
os.path.join(extraPrefix,
os.path.basename(tomo.getFileName())))
newTi.setSamplingRate(
self.inputSetOfTomograms.get().getSamplingRate())
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
# Set origin to output tilt-series
origin.setShifts(
x / -2. * self.inputSetOfTomograms.get().getSamplingRate(),
y / -2. * self.inputSetOfTomograms.get().getSamplingRate(), 0)
newTs.setOrigin(origin)
newTs.write(properties=False)
outputProjectedSetOfTiltSeries.update(newTs)
outputProjectedSetOfTiltSeries.updateDim()
outputProjectedSetOfTiltSeries.write()
self._store()
def resizeTiltSeries(self, tsObjId):
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
path.makePath(extraPrefix)
args = self._resizeCommonArgs(self)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(ts.getFirstItem().getFileName())
ih.createEmptyImage(fnOut=os.path.join(
extraPrefix,
ts.getFirstItem().parseFileName()),
xDim=int(x * self.factor),
yDim=int(y * self.factor),
nDim=1)
for ti in ts:
self.runJob("xmipp_image_resize", self._ioArgs(ti) + args)
def createOutputStep(self, tsObjId):
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
outputSetOfTiltSeries = self.getOutputSetOfTiltSeries()
newTs = tomoObj.TiltSeries(tsId=tsId)
newTs.copyInfo(ts)
outputSetOfTiltSeries.append(newTs)
newTs.setSamplingRate(self.samplingRate)
fileName = ts.getFirstItem().parseFileName()
for index, ti in enumerate(ts):
newTi = tomoObj.TiltImage()
newTi.copyInfo(ti, copyId=True)
newTi.setLocation(index + 1, os.path.join(extraPrefix, fileName))
if ti.hasTransform():
newTi.setTransform(ti.getTransform())
newTi.setSamplingRate(self.samplingRate)
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
newTs.write(properties=False)
outputSetOfTiltSeries.update(newTs)
outputSetOfTiltSeries.updateDim()
outputSetOfTiltSeries.write()
self._store()
def _findVesicleCenter(self, starFileInit, starFilePreseg1):
ih = ImageHandler()
outputTable = self._createTable()
# Read preseg (vesicles not centered) table
presegTable = Table()
presegTable.read(starFilePreseg1)
# Read initial data
initTable = Table()
initTable.read(starFileInit)
# Generate the star file with the vesicles centered for the second pre_seg execution
for row, rowp in zip(initTable, presegTable):
tomo = row.get(TOMOGRAM, NOT_FOUND)
vesicle = row.get(VESICLE, NOT_FOUND)
materialIndex = row.get(PYSEG_LABEL, NOT_FOUND)
# Get the upper left corner of the vesicle in the original tomogram: _psSegOffX #7 _psSegOffY #8 _psSegOffZ
# #9 from output star file of pre_tomos_seg.py that do not distinguish inner and outer densities
xdimCorner = rowp.get(PYSEG_OFFSET_X, 0)
ydimCorner = rowp.get(PYSEG_OFFSET_Y, 0)
zdimCorner = rowp.get(PYSEG_OFFSET_Z, 0)
# Get the box dimensions, be sure that the Dimensions format is Dimensions: 239 x 1 x 298 x 298
# ((N)Objects x (Z)Slices x (Y)Rows x (X)Columns)
x, y, z, _ = ih.getDimensions(rowp.get(MASK, NOT_FOUND))
# Add row to output table
outputTable.addRow(
tomo,
vesicle,
materialIndex,
vesicle,
xdimCorner + x / 2,
ydimCorner + y / 2,
zdimCorner + z / 2,
)
outputTable.write(self.getVesiclesCenteredStarFile())
def _validate(self):
ci = self.getImportClass()
if ci is None:
errors = ProtImportMicBase._validate(self)
for micFn, _ in self.iterFiles():
imgh = ImageHandler()
if imgh.isImageFile(micFn):
_, _, z, n = imgh.getDimensions(micFn)
if n > 1 or z > 1:
errors.append("The protocol not support micrographs "
"stored in stacks. If you want to "
"obtain your micrographs individually, "
"you can run the following command:\n"
"scipion run scipion_directory/scripts/"
"split_stacks.py --files *your files* "
"--ext *extension*")
# JMRT: only check the first image, for large dataset
# even reading the header can take a while
break
return errors
else:
return ci.validateMicrographs()
def __applyTransform(suffix, pdbFileName, shift, angles, sampling):
""" auxiliary function, transform PDB and 3dmap files"""
# create a Scipion transformation matrix
from numpy import deg2rad
rotation_matrix = emconv.euler_matrix(deg2rad(angles[0]),
deg2rad(angles[1]),
deg2rad(angles[2]), 'szyz')
translation = emconv.translation_matrix(shift)
M = emconv.concatenate_matrices(rotation_matrix, translation)
# apply it to the pdb file
# if rotation move to center
aSH = emconv.AtomicStructHandler(pdbFileName)
if angles[0] != 0. or angles[1] != 0. or angles[2] != 0.:
ih = ImageHandler()
x, y, z, n = ih.getDimensions("emd_%s.map" % EMDBID)
x /= 2.
y /= 2.
z /= 2.
localShift = [-x, -y, -z]
rotation_matrix = emconv.euler_matrix(0., 0., 0., 'szyz')
translation = emconv.translation_matrix(localShift)
localM = emconv.concatenate_matrices(rotation_matrix,
translation)
aSH.transform(localM, sampling=sampling)
aSH.transform(M, sampling=sampling)
if angles[0] != 0. or angles[1] != 0. or angles[2] != 0.:
localShift = [x, y, z]
rotation_matrix = emconv.euler_matrix(0., 0., 0., 'szyz')
translation = emconv.translation_matrix(localShift)
localM = emconv.concatenate_matrices(rotation_matrix,
translation)
aSH.transform(localM, sampling=sampling)
aSH.write("%s_%s_transformed.ent" % (suffix, PDBID.lower()))
# get equivalent xmipp transformation
shift, angles = __getXmippEulerAngles(M)
# shift 3map and set sampling
__runXmippProgram(
"xmipp_transform_geometry", '-i emd_%s.map '
'-o %s_emd_%s_transform.map '
'--interp linear '
'--shift %f %f %f '
'--rotate_volume euler %f %f %f ' %
(EMDBID, suffix, EMDBID, shift[0], shift[1], shift[2],
angles[0], angles[1], angles[2]))
header = Ccp4Header("%s_emd_%s_transform.map" % (suffix, EMDBID),
readHeader=True)
header.setSampling(sampling)
# put the sampling back, xmipp_transform_geometry erased it
header.writeHeader()
# view the results with chimera
from pwem.viewers import Chimera
args = "%s %s %s %s" % (
pdbFileName, "emd_%s.map" % EMDBID, "%s_%s_transformed.ent" %
(suffix, PDBID.lower()), "%s_emd_%s_transform.map" %
(suffix, EMDBID))
Chimera.runProgram(args)
def getResizeFactorFromDimensions(self, outputDim):
ih = ImageHandler()
originalDim, _, _, _ = ih.getDimensions(
self.inputTiltSeries.get().getFirstItem().getFileName())
return round(outputDim / originalDim)
def getPixSizeFromDimensions(self, outputDim):
ih = ImageHandler()
originalDim, _, _, _ = ih.getDimensions(
self.inputTiltSeries.getFirstItem().getFileName())
return self.inputTiltSeries.getSamplingRate() * round(
originalDim / outputDim)
def createOutput(self):
outputPrealiSetOfTiltSeries = None
outputAliSetOfTiltSeries = None
outputSetOfLandmarkModelsNoGaps = None
outputSetOfCoordinates3D = None
outputSetOfFullTomograms = None
outputSetOfPostProcessTomograms = None
for ts in self.inputSetOfTiltSeries.get():
self.inputTiltSeries = ts
tsId = ts.getTsId()
"""Prealigned tilt-series"""
prealiFilePath = self.getFilePath(ts, extension=".preali")
if os.path.exists(prealiFilePath):
if outputPrealiSetOfTiltSeries is None:
outputPrealiSetOfTiltSeries = self._createSetOfTiltSeries(
suffix='Preali')
outputPrealiSetOfTiltSeries.copyInfo(self.inputTiltSeries)
outputPrealiSetOfTiltSeries.setDim(
self.inputTiltSeries.getDim())
self._defineOutputs(outputPrealignedSetOfTiltSeries=
outputPrealiSetOfTiltSeries)
self._defineSourceRelation(self.inputTiltSeries,
outputPrealiSetOfTiltSeries)
newTs = ts.clone()
newTs.copyInfo(ts)
outputPrealiSetOfTiltSeries.append(newTs)
ih = ImageHandler()
index = 0
for index, tiltImage in enumerate(ts.iterItems(iterate=False)):
newTi = tiltImage.clone()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(index + 1, prealiFilePath)
index += 1
xPreali, _, _, _ = ih.getDimensions(newTi.getFileName() +
":mrc")
newTi.setSamplingRate(
self.getPixSizeFromDimensions(xPreali))
newTs.append(newTi)
xPreali, yPreali, zPreali, _ = ih.getDimensions(
newTs.getFirstItem().getFileName() + ":mrc")
newTs.setDim((xPreali, yPreali, zPreali))
newTs.write(properties=False)
outputPrealiSetOfTiltSeries.setSamplingRate(
self.getPixSizeFromDimensions(xPreali))
outputPrealiSetOfTiltSeries.write()
self._store(outputPrealiSetOfTiltSeries)
"""Aligned tilt-series"""
aligFilePath = self.getFilePath(ts, extension=".ali")
if os.path.exists(aligFilePath):
if outputAliSetOfTiltSeries is None:
outputAliSetOfTiltSeries = self._createSetOfTiltSeries(
suffix='Ali')
outputAliSetOfTiltSeries.copyInfo(self.inputTiltSeries)
outputAliSetOfTiltSeries.setDim(
self.inputTiltSeries.getDim())
self._defineOutputs(
outputAlignedSetOfTiltSeries=outputAliSetOfTiltSeries)
self._defineSourceRelation(self.inputSetOfTiltSeries,
outputAliSetOfTiltSeries)
newTs = ts.clone()
newTs.copyInfo(ts)
outputAliSetOfTiltSeries.append(newTs)
ih = ImageHandler()
tltFilePath = self.getFilePath(ts,
suffix='_fid',
extension=".tlt")
if os.path.exists(tltFilePath):
tltList = utils.formatAngleList(tltFilePath)
else:
tltList = None
index = 0
for index, tiltImage in enumerate(ts.iterItems(iterate=False)):
newTi = tiltImage.clone()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(index + 1, aligFilePath)
if tltList is not None:
newTi.setTiltAngle(float(tltList[index]))
xAli, _, _, _ = ih.getDimensions(newTi.getFileName() +
":mrc")
newTi.setSamplingRate(self.getPixSizeFromDimensions(xAli))
newTs.append(newTi)
xAli, yAli, zAli, _ = ih.getDimensions(
newTs.getFirstItem().getFileName() + ":mrc")
newTs.setDim((xAli, yAli, zAli))
newTs.write(properties=False)
outputAliSetOfTiltSeries.setSamplingRate(
self.getPixSizeFromDimensions(xAli))
outputAliSetOfTiltSeries.write()
self._store(outputAliSetOfTiltSeries)
"""Output set of coordinates 3D (associated to the aligned tilt-series)"""
coordFilePath = self.getFilePath(ts,
suffix='fid',
extension=".xyz")
if os.path.exists(coordFilePath):
if outputSetOfCoordinates3D is None:
outputSetOfCoordinates3D = self._createSetOfCoordinates3D(
volSet=outputAliSetOfTiltSeries,
suffix='LandmarkModel')
outputSetOfCoordinates3D.setSamplingRate(
self.inputTiltSeries.getSamplingRate())
outputSetOfCoordinates3D.setPrecedents(
outputAliSetOfTiltSeries)
self._defineOutputs(
outputSetOfCoordinates3D=outputSetOfCoordinates3D)
self._defineSourceRelation(self.inputSetOfTiltSeries,
outputSetOfCoordinates3D)
coordList = utils.format3DCoordinatesList(coordFilePath)
for element in coordList:
newCoord3D = tomoObj.Coordinate3D()
newCoord3D.setVolume(ts)
newCoord3D.setX(element[0], constants.BOTTOM_LEFT_CORNER)
newCoord3D.setY(element[1], constants.BOTTOM_LEFT_CORNER)
newCoord3D.setZ(element[2], constants.BOTTOM_LEFT_CORNER)
newCoord3D.setVolId(ts.getObjId())
outputSetOfCoordinates3D.append(newCoord3D)
outputSetOfCoordinates3D.update(newCoord3D)
outputSetOfCoordinates3D.write()
self._store(outputSetOfCoordinates3D)
"""Landmark models with no gaps"""
if (os.path.exists(
self.getFilePath(ts, suffix="_nogaps", extension=".fid"))
and os.path.exists(self.getFilePath(ts,
extension=".resid"))):
paramsNoGapPoint2Model = {
'inputFile':
self.getFilePath(ts, suffix="_nogaps", extension=".fid"),
'outputFile':
self.getFilePath(ts,
suffix="_nogaps_fid",
extension=".txt")
}
argsNoGapPoint2Model = "-InputFile %(inputFile)s " \
"-OutputFile %(outputFile)s"
Plugin.runImod(self, 'model2point',
argsNoGapPoint2Model % paramsNoGapPoint2Model)
if outputSetOfLandmarkModelsNoGaps is None:
outputSetOfLandmarkModelsNoGaps = self._createSetOfLandmarkModels(
suffix='NoGaps')
outputSetOfLandmarkModelsNoGaps.copyInfo(
self.inputTiltSeries)
self._defineOutputs(outputSetOfLandmarkModelsNoGaps=
outputSetOfLandmarkModelsNoGaps)
self._defineSourceRelation(
self.inputSetOfTiltSeries,
outputSetOfLandmarkModelsNoGaps)
fiducialNoGapFilePath = self.getFilePath(ts,
suffix="_nogaps_fid",
extension=".txt")
fiducialNoGapList = utils.formatFiducialList(
fiducialNoGapFilePath)
fiducialModelNoGapPath = self.getFilePath(ts,
suffix="_nogaps",
extension=".fid")
landmarkModelNoGapsFilePath = self.getFilePath(
ts, suffix="_nogaps", extension=".sfid")
landmarkModelNoGapsResidPath = self.getFilePath(
ts, extension=".resid")
fiducialNoGapsResidList = utils.formatFiducialResidList(
landmarkModelNoGapsResidPath)
landmarkModelNoGaps = tomoObj.LandmarkModel(
tsId, landmarkModelNoGapsFilePath, fiducialModelNoGapPath)
prevTiltIm = 0
chainId = 0
indexFake = 0
for fiducial in fiducialNoGapList:
if int(float(fiducial[2])) <= prevTiltIm:
chainId += 1
prevTiltIm = int(float(fiducial[2]))
if (indexFake < len(fiducialNoGapsResidList)
and fiducial[2]
== fiducialNoGapsResidList[indexFake][2]):
landmarkModelNoGaps.addLandmark(
xCoor=fiducial[0],
yCoor=fiducial[1],
tiltIm=fiducial[2],
chainId=chainId,
xResid=fiducialNoGapsResidList[indexFake][3],
yResid=fiducialNoGapsResidList[indexFake][4])
indexFake += 1
else:
landmarkModelNoGaps.addLandmark(xCoor=fiducial[0],
yCoor=fiducial[1],
tiltIm=fiducial[2],
chainId=chainId,
xResid='0',
yResid='0')
outputSetOfLandmarkModelsNoGaps.append(landmarkModelNoGaps)
outputSetOfLandmarkModelsNoGaps.write()
self._store(outputSetOfLandmarkModelsNoGaps)
"""Full reconstructed tomogram"""
reconstructTomoFilePath = self.getFilePath(ts,
suffix="_full",
extension=".rec")
if os.path.exists(reconstructTomoFilePath):
if outputSetOfFullTomograms is None:
outputSetOfFullTomograms = self._createSetOfTomograms(
suffix='Full')
outputSetOfFullTomograms.copyInfo(self.inputTiltSeries)
self._defineOutputs(
outputSetOfFullTomograms=outputSetOfFullTomograms)
self._defineSourceRelation(self.inputSetOfTiltSeries,
outputSetOfFullTomograms)
newTomogram = tomoObj.Tomogram()
newTomogram.setLocation(reconstructTomoFilePath)
newTomogram.setSamplingRate(ts.getSamplingRate())
outputSetOfFullTomograms.append(newTomogram)
outputSetOfFullTomograms.write()
self._store(outputSetOfFullTomograms)
"""Post-processed reconstructed tomogram"""
posprocessedRecTomoFilePath = self.getFilePath(ts,
extension=".rec")
if os.path.exists(posprocessedRecTomoFilePath):
if outputSetOfPostProcessTomograms is None:
outputSetOfPostProcessTomograms = self._createSetOfTomograms(
)
outputSetOfPostProcessTomograms.copyInfo(
self.inputTiltSeries)
self._defineOutputs(outputSetOfPostProcessTomograms=
outputSetOfPostProcessTomograms)
self._defineSourceRelation(
self.inputSetOfTiltSeries,
outputSetOfPostProcessTomograms)
newTomogram = tomoObj.Tomogram()
newTomogram.setLocation(posprocessedRecTomoFilePath)
outputSetOfPostProcessTomograms.append(newTomogram)
outputSetOfPostProcessTomograms.write()
self._store(outputSetOfPostProcessTomograms)
self.closeMappers()
class ReportInflux:
""" Create a report (html or influxdb) with a summary of the processing.
The report will be updated with a given frequency.
"""
def __init__(self,
protocol,
ctfMonitor,
sysMonitor,
movieGainMonitor,
publishCmd=None,
**kwargs):
"""
:param protocol:
:param ctfMonitor:
:param sysMonitor:
:param movieGainMonitor:
:param publishCmd: Send it to a server computer so user
may access this information via web
:param influxdb: False -> scipion should create a web page
True -> scipion should connect to a database
:param kwargs: refreshSecs -> update each refreshSecs seconds
"""
# The CTF protocol to monitor
self.protocol = protocol
self.ctfProtocol = protocol._getCtfProtocol()
self.alignProtocol = protocol._getAlignProtocol()
self.micThumbSymlinks = False
self.reportPath = protocol.reportPath
self.reportDir = protocol.reportDir
self.provider = SummaryProvider(protocol)
self.ctfMonitor = ctfMonitor
self.sysMonitor = sysMonitor
self.movieGainMonitor = movieGainMonitor
self.lastThumbIndex = 0
self.thumbsReady = 0
self.publishCmd = publishCmd
self.refreshSecs = kwargs.get('refreshSecs', 60)
if self.refreshSecs < 10:
self.refreshSecs = 10
self.ih = ImageHandler()
# Create a config file to store data we are going to
# need in other iteration. So far is only used by influx
# but it certainly be used by hte standard html
# for example to decide which files need to be tranfered
self.confFileName = self.protocol._getTmpPath(CONFILE)
if not os.path.isfile(self.confFileName):
# Create the configuration file as it doesn't exist yet
self.confParser = ConfigParser()
self.confParser.add_section("project")
self.confParser.set("project", "projectName",
self.protocol.getProject().getShortName())
self.confParser.set("project", "properties", "1")
self.confParser.set("project", "acquisition", "1")
self.confParser.set("project", "summary", "1")
self.confParser.add_section('ctf')
self.confParser.set("ctf", "lastId", "0")
self.confParser.add_section("gain")
self.confParser.set("gain", "lastId", "0")
self.confParser.add_section("system")
self.confParser.set("system", "lastId", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
else:
self.confParser = ConfigParser()
self.confParser.read(self.confFileName)
# open connection to database
# I put this import here so users with no database
# can run tratinional html report
from influxdb import InfluxDBClient
confParser = ConfigParser()
from emfacilities.constants import (SECRETSFILE,
EMFACILITIES_HOME_VARNAME)
_path = os.getenv(EMFACILITIES_HOME_VARNAME)
secretsfile = os.path.join(_path, SECRETSFILE)
confParser.read(secretsfile)
#influx data
dataBase = confParser.get('influx', 'dataBase')
passwordInflux = confParser.get('influx', 'passwordInflux')
usernameInflux = confParser.get('influx', 'usernameInflux')
hostinflux = confParser.get('influx', 'hostinflux')
port = int(confParser.get('influx', 'port'))
ssl = confParser.get('influx', 'ssl')
ssl = ssl.lower() in ['true', 1, 't', 'y']
verify_ssl = confParser.get('influx', 'verify_ssl')
verify_ssl = verify_ssl.lower() in ['true', 1, 't', 'y']
self.timeDelta = int(confParser.get('influx', 'TimeDelta'))
# directory with images from apaches point of view
self.apacheImgDir = confParser.get('influx', 'apacheImgDir')
self.dataBaseName = dataBase
#paramiko data
self.usernameParamiko = confParser.get('paramiko', 'usernameParamiko')
self.passwordParamiko = confParser.get('paramiko', 'passwordParamiko')
self.keyfilepath = confParser.get('paramiko', 'keyfilepath')
self.keyfiletype = confParser.get('paramiko', 'keyfiletype')
self.remote_path = confParser.get('paramiko', 'remote_path')
self.hostparamiko = confParser.get('paramiko', 'hostparamiko')
try:
# since I am using a self generated certificate
# the certificate can not be verified against a
# certificate authority. So I disable the warnings
urllib3.disable_warnings()
# open conection to influx
self.client = InfluxDBClient(host=hostinflux,
port=port,
username=deCrypt(usernameInflux),
password=deCrypt(passwordInflux),
ssl=ssl,
verify_ssl=verify_ssl)
# we can only create the database here if scipion_writer
# has admin privileges
# If you attempt to create a database that already exists,
# InfluxDB does nothing and does not return an error.
# self.client.create_database("scipion")
self.client.switch_database(self.dataBaseName)
self.projectName = slugify(
self.protocol.getProject().getShortName())
# delete meassurement
# project names may contain forbiden character
# IF this is a problem we will need to slugify the projName
# self.client.drop_measurement(self.projectName)
self.client.delete_series(measurement=self.projectName)
print("dropping meassurement:", self.projectName)
# replication -> number of copies of the DB stored in the cluster
# 12w -> delete data after 12 weeks
self.client.create_retention_policy("ret_pol",
"12w",
replication=1,
default=True)
except Exception as e:
print("Error:", e)
#def __del__(self):
# if self.influxsb:
# self.client.close()
def generate(self, finished):
project = self.protocol.getProject()
# Project Properties Section
# Do not delete this variables. We are using them
# in an eval command
self.projectName = project.getShortName()
startTime = pwutils.dateStr(project.getCreationTime(), secs=True),
tnow = datetime.now()
_now = project.getCreationTime()
dateStr = pwutils.prettyTime(dt=tnow, secs=True),
projectDuration = pwutils.prettyDelta(tnow -
project.getCreationTime()),
projectStatus = "FINISHED" if finished else "RUNNING",
scipionVersion = os.environ['SCIPION_VERSION'],
# create structure with information related with the properties section
# the primmary key is time, projectionName, section
# if we insert two points with these three values idential
# the second updates the first
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'properties'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "properties")
fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2}
fieldNames = {
'dateStr': "<b>Last Update</b>",
'projectDuration': '<b>Duration</b>',
'projectStatus': '<b>Status</b>'
}
if firstTime:
fieldKeys['startTime'] = 5
fieldKeys['scipionVersion'] = 1
fieldNames['startTime'] = '<b>Start Time</b>'
fieldNames['scipionVersion'] = '<b>Scipion Version</b>'
for metric, delta in fieldKeys.items():
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = fieldNames[metric]
# str is need because all values must have the same type
fields['valueStr'] = str(eval(metric)[0])
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
self.confParser.set("project", "properties", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# acquisition section
self.provider.refreshObjects()
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'acquisition'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "acquisition")
if firstTime:
delta = 0
for metricName, value in self.provider.acquisition:
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = "<b>" + metricName + "</b>"
fields['valueNum'] = float(value)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
delta = delta - 1
# update first time only if some date has been read.
# do not place this upside the loop
self.confParser.set("project", "acquisition", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# send summary section
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'summary'
pointsDict['tags'] = tags
localNow = _now
for obj in self.provider.getObjects():
fields = {}
localNow = localNow + timedelta(seconds=-1)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
fields['protocolName'] = '<b>' + obj.name + '</b>'
fields['output'] = ""
fields['size'] = ""
else:
fields['protocolName'] = ""
fields['output'] = obj.output
fields['size'] = str(obj.outSize)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
# Ctf monitor chart data
last_id = self.confParser.getint("ctf", "lastId")
listDictionaryCTF = {} if self.ctfMonitor is None else \
self.ctfMonitor.getData(lastId=last_id)
if listDictionaryCTF:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'ctf'
for ctf in listDictionaryCTF:
fields = {}
for key in ctf:
if key == 'timestamp':
pointsDict['time'] = ctf['timestamp']
# id must be a tag since tw CTF may have the same timestamp
# and time + tags must be unique
elif key == 'id':
tags['id'] = ctf['id']
# elif key == 'ctfID':
# fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>"
elif key == 'shiftPlotPath':
temp = os.path.join(self.apacheImgDir,
self.projectName,
basename(ctf[key]))
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, basename(ctf[key]))
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
elif key == 'psdPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
elif key == 'micPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
else:
fields[key] = ctf[key]
# while be use to control image creation
fields["transferImage"] = False
pointsDict['fields'] = fields
pointsDict['tags'] = tags
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("ctf", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# GAIN Section
last_id = self.confParser.getint("gain", "lastId")
listDictionaryGain = {} if self.movieGainMonitor is None else \
self.movieGainMonitor.getData(lastId=last_id)
if listDictionaryGain:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'gain'
pointsDict['tags'] = tags
counter = 1
tnow = datetime.now()
for gain in listDictionaryGain:
fields = {}
for key in gain:
fields[key] = gain[key]
pointsDict['fields'] = fields
# gain has no time information so I just
# put current time
localNow = tnow + timedelta(seconds=counter)
pointsDict[
'time'] = localNow # .strftime('%Y-%m-%dT%H:%M:%SZ')
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("gain", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# SYSTEM data
last_id = self.confParser.getint("system", "lastId")
listDictionarySystem = {} if self.sysMonitor is None else \
self.sysMonitor.getData(lastId=last_id)
if listDictionarySystem:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'system'
pointsDict['tags'] = tags
for system in listDictionarySystem:
fields = {}
for key in system:
if key == 'timestamp':
pointsDict['time'] = system['timestamp']
elif key == 'id':
continue
else:
fields[key] = system[key]
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("system", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
self.transferFiles()
return last_id # reportFinished
def transferFiles(self):
# get images that need to be transfered
# in grupos of 10 images
start_time = time.time()
query = '''select *
from "%s"
where section = 'ctf'
and transferImage = false
order by time desc
limit 10''' % self.projectName
result = self.client.query(query)
if len(result) < 1:
return True
connect = Connect(host=self.hostparamiko,
port=22,
username=deCrypt(self.usernameParamiko),
password=None,
keyfilepath=deCrypt(self.keyfilepath),
keyfiletype=deCrypt(self.keyfiletype),
remote_path=self.remote_path,
projectName=self.projectName)
while True:
result = self.client.query(query)
source = []
target = []
newPoints = []
points = result.get_points()
for point in points:
tags = {}
tags['section'] = 'ctf'
pointsDict = {}
source.append(point['shiftPlotPathLocal']) # shift plot
source.append(point['psdPathLocalPng']) # psd image
source.append(point['micPathLocalPng']) # micrograph
# default value 512
try:
X, Y, Z, N = self.ih.getDimensions(point['psdPathLocal'])
except:
print("file %s does not exist" % point['psdPathLocal'])
if X > 512:
scaleFactor = X // 512
else:
scaleFactor = 1
self.ih.computeThumbnail(point['psdPathLocal'],
point['psdPathLocalPng'],
scaleFactor=scaleFactor,
flipOnY=True)
X, Y, Z, N = self.ih.getDimensions(point['micPathLocal'])
if X > 512:
scaleFactor = X // 512
else:
scaleFactor = 1
self.ih.computeThumbnail(point['micPathLocal'],
point['micPathLocalPng'],
scaleFactor=scaleFactor,
flipOnY=True)
# add files to transfer list
target.append(
os.path.join(self.projectName,
basename(point['shiftPlotPathLocal'])))
target.append(
os.path.join(self.projectName,
basename(point['psdPathLocalPng'])))
target.append(
os.path.join(self.projectName,
basename(point['micPathLocalPng'])))
#
point['transferImage'] = True
pointsDict['time'] = point['time']
tags['id'] = point['id']
del point['time']
del point['section']
del point['id']
pointsDict['fields'] = point
pointsDict['tags'] = tags
pointsDict['measurement'] = self.projectName
newPoints.append(pointsDict)
try:
connect.put(source, target)
except Exception as e:
print("Error tranfering files: ", e)
return False # do not mark files are read
# update influx
try:
self.client.write_points(points=newPoints)
except Exception as e:
print("Error updating database: ", e)
return False # do not mark files are read
elapsed_time = time.time() - start_time
if elapsed_time > self.refreshSecs:
break
elif len(result) == 0:
break
connect.close()
return True
def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
amplitudeContrast, magnification):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
imgSet = self._getOutputSet() if self.isContinued() else None
self.importedFiles = set()
if imgSet is None:
createSetFunc = getattr(self, '_create' + self._outputClassName)
imgSet = createSetFunc()
elif imgSet.getSize() > 0: # in case of continue
imgSet.loadAllProperties()
self._fillImportedFiles(imgSet)
imgSet.enableAppend()
pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
if pointerExcludedMovs is not None:
excludedMovs = pointerExcludedMovs.get()
self._fillImportedFiles(excludedMovs)
imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
acquisition = imgSet.getAcquisition()
self.fillAcquisition(acquisition)
# Call a function that should be implemented by each subclass
self.setSamplingRate(imgSet)
outFiles = [imgSet.getFileName()]
imgh = ImageHandler()
img = imgSet.ITEM_TYPE()
img.setAcquisition(acquisition)
n = 1
copyOrLink = self.getCopyOrLink()
outputName = self._getOutputName()
alreadyWarned = False # Use this flag to warn only once
finished = False
# this is only used when creating stacks from frame files
self.createdStacks = set()
i = 0
lastDetectedChange = datetime.now()
# Ignore the timeout variables if we are not really in streaming mode
if self.dataStreaming:
timeout = timedelta(seconds=self.timeout.get())
fileTimeout = timedelta(seconds=self.fileTimeout.get())
else:
timeout = timedelta(seconds=5)
fileTimeout = timedelta(seconds=5)
while not finished:
time.sleep(3) # wait 3 seconds before check for new files
someNew = False
someAdded = False
for fileName, uniqueFn, fileId in self.iterNewInputFiles():
someNew = True
if self.fileModified(fileName, fileTimeout):
continue
dst = self._getExtraPath(uniqueFn)
self.importedFiles.add(uniqueFn)
dst, alreadyWarned = cleanFileName(dst, not alreadyWarned)
copyOrLink(fileName, dst)
self.debug('Importing file: %s' % fileName)
self.debug("uniqueFn: %s" % uniqueFn)
self.debug("dst Fn: %s" % dst)
if self._checkStacks:
_, _, _, n = imgh.getDimensions(dst)
someAdded = True
self.debug('Appending file to DB...')
if self.importedFiles: # enable append after first append
imgSet.enableAppend()
if n > 1:
for index in range(1, n + 1):
img.cleanObjId()
img.setMicId(fileId)
img.setFileName(dst)
img.setIndex(index)
self._addImageToSet(img, imgSet)
else:
img.setObjId(fileId)
img.setFileName(dst)
# Fill the micName if img is either a Micrograph or a Movie
uniqueFn = uniqueFn.replace(' ', '')
self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
self._fillMicName(img, uniqueFn)
self._addImageToSet(img, imgSet)
outFiles.append(dst)
self.debug('After append. Files: %d' % len(outFiles))
if someAdded:
self.debug('Updating output...')
self._updateOutputSet(outputName,
imgSet,
state=imgSet.STREAM_OPEN)
self.debug('Update Done.')
self.debug('Checking if finished...someNew: %s' % someNew)
now = datetime.now()
if not someNew:
# If there are no new detected files, we should check the
# inactivity time elapsed (from last event to now) and
# if it is greater than the defined timeout, we conclude
# the import and close the output set
# Another option is to check if the protocol have some
# special stop condition, this can be used to manually stop
# some protocols such as import movies
finished = (now - lastDetectedChange > timeout
or self.streamingHasFinished())
self.debug("Checking if finished:")
self.debug(" Now - Last Change: %s" %
pwutils.prettyDelta(now - lastDetectedChange))
self.debug("Finished: %s" % finished)
else:
# If we have detected some files, we should update
# the timestamp of the last event
lastDetectedChange = now
self._updateOutputSet(outputName, imgSet, state=imgSet.STREAM_CLOSED)
self._cleanUp()
return outFiles
def test_tomoReconstructionOutputTomogramDimensions(self):
ih = ImageHandler()
outputDimensions = ih.getDimensions(
self.protTomoReconstruction.outputSetOfTomograms.getFirstItem())
self.assertTrue(outputDimensions == (512, 512, self.thicknessTomo, 1))
def _loadTable(self, tableName):
""" Load information from tables PREFIX_Classes, PREFIX_Objects. """
tableName = self.tablePrefixes[tableName]
BASIC_COLUMNS = [
Column('id', int, renderType=COL_RENDER_ID),
Column('enabled', bool, renderType=COL_RENDER_CHECKBOX),
Column('label', str),
Column('comment', str),
Column('creation', str)
]
# Load columns from PREFIX_Classes table
columns = list(BASIC_COLUMNS)
db = SqliteDb()
db._createConnection(self._dbName, 1000)
db.executeCommand("SELECT * FROM %sClasses;" % tableName)
# This will store the images columns to join
# the _index and the _filename
imgCols = {}
for row in db._iterResults():
renderType = COL_RENDER_NONE
colName = row['column_name']
colLabel = row['label_property']
if colLabel != 'self':
# Keep track of _index and _filename pairs to mark as renderable images
if colLabel.endswith('_index'):
imgCols[colLabel.replace('_index', '')] = colName
elif colLabel.endswith('_filename'):
# TODO: Maybe not all the labels endswith "_filename"
# have to be rendered.
# for example in the RotSpectra with '_representative._filename'
prefix = colLabel.replace('_filename', '')
if prefix in imgCols:
renderType = COL_RENDER_IMAGE
imgCols[colName] = imgCols[prefix]
# CTF FIX
elif (colLabel.endswith('_psdFile')
or colLabel.endswith('_enhanced_psd')
or colLabel.endswith('_ctfmodel_quadrant')
or colLabel.endswith('_ctfmodel_halfplane')):
renderType = COL_RENDER_IMAGE
if row['class_name'] == 'Boolean':
renderType = COL_RENDER_CHECKBOX
columns.append(
Column(colName, str, label=colLabel,
renderType=renderType))
table = Table(*columns)
checkedImgCols = {} # Check if the image columns are volumes
# FIXME: Move this to scipion-em? Maybe remove the whole module that is not used?
from pwem.emlib.image import ImageHandler
ih = ImageHandler()
# Populate the table in the DataSet
db.executeCommand("SELECT * FROM %sObjects;" % tableName)
for row in db._iterResults():
rowDict = dict(row)
for k, v in rowDict.items():
if v is None:
rowDict[k] = ''
# Set the index@filename for images columns values
if k in imgCols:
colName = imgCols[k]
index = rowDict[colName]
filename = os.path.join(self.projectPath, rowDict[k])
filepath = filename.replace(":mrc", "")
if not checkedImgCols.get(colName, False):
if os.path.exists(filepath):
# print "Fn to get dims: %s@%s" % (index,filename)
x, y, z, n = ih.getDimensions((index, filename))
if z > 1:
table.getColumn(k).setRenderType(
COL_RENDER_VOLUME)
checkedImgCols[colName] = True
if index:
rowDict[k] = '%06d@%s' % (index, filename)
table.addRow(row['id'], **rowDict)
return table
def importImagesStep(self, pattern, voltage, sphericalAberration,
amplitudeContrast, magnification):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
createSetFunc = getattr(self, '_create' + self._outputClassName)
imgSet = createSetFunc()
imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
acquisition = imgSet.getAcquisition()
self.fillAcquisition(acquisition)
# Call a function that should be implemented by each subclass
self.setSamplingRate(imgSet)
outFiles = [imgSet.getFileName()]
imgh = ImageHandler()
img = imgSet.ITEM_TYPE()
img.setAcquisition(acquisition)
n = 1
copyOrLink = self.getCopyOrLink()
alreadyWarned = False # Use this flag to warn only once
for i, (fileName, fileId) in enumerate(self.iterFiles()):
if self.isBlacklisted(fileName):
continue
uniqueFn = self._getUniqueFileName(fileName)
dst = self._getExtraPath(uniqueFn)
if ' ' in dst:
if not alreadyWarned:
self.warning('Warning: your file names have white spaces!')
self.warning('Removing white spaces from copies/symlinks.')
alreadyWarned = True
dst = dst.replace(' ', '')
copyOrLink(fileName, dst)
# Handle special case of Imagic images, copying also .img or .hed
self.handleImgHed(copyOrLink, fileName, dst)
if self._checkStacks:
_, _, _, n = imgh.getDimensions(dst)
if n > 1:
for index in range(1, n + 1):
img.cleanObjId()
img.setMicId(fileId)
img.setFileName(dst)
img.setIndex(index)
self._addImageToSet(img, imgSet)
else:
img.setObjId(fileId)
img.setLocation(dst)
# Fill the micName if img is either Micrograph or Movie
uniqueFn = uniqueFn.replace(' ', '')
self._fillMicName(img, uniqueFn)
self._addImageToSet(img, imgSet)
outFiles.append(dst)
sys.stdout.write("\rImported %d/%d\n" %
(i + 1, self.numberOfFiles))
sys.stdout.flush()
print("\n")
args = {}
outputSet = self._getOutputName()
args[outputSet] = imgSet
self._defineOutputs(**args)
return outFiles
def generateOutputStackStep(self, tsObjId):
outputInterpolatedSetOfTiltSeries = self.getOutputInterpolatedSetOfTiltSeries(
)
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
firstItem = ts.getFirstItem()
paramsAlignment = {
'input':
firstItem.getFileName(),
'output':
os.path.join(extraPrefix, firstItem.parseFileName()),
'xform':
os.path.join(extraPrefix,
firstItem.parseFileName(extension=".prexg")),
'bin':
int(self.binning.get()),
'imagebinned':
1.0
}
argsAlignment = "-input %(input)s " \
"-output %(output)s " \
"-xform %(xform)s " \
"-bin %(bin)d " \
"-imagebinned %(imagebinned)s "
rotationAngleAvg = utils.calculateRotationAngleFromTM(ts)
# Check if rotation angle is greater than 45º. If so, swap x and y dimensions to adapt output image sizes to
# the final sample disposition.
if rotationAngleAvg > 45 or rotationAngleAvg < -45:
paramsAlignment.update(
{'size': "%d,%d" % (firstItem.getYDim(), firstItem.getXDim())})
argsAlignment += "-size %(size)s "
Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
newTs = tomoObj.TiltSeries(tsId=tsId)
newTs.copyInfo(ts)
outputInterpolatedSetOfTiltSeries.append(newTs)
if self.binning > 1:
newTs.setSamplingRate(ts.getSamplingRate() *
int(self.binning.get()))
for index, tiltImage in enumerate(ts):
newTi = tomoObj.TiltImage()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(
index + 1,
(os.path.join(extraPrefix, tiltImage.parseFileName())))
if self.binning > 1:
newTi.setSamplingRate(tiltImage.getSamplingRate() *
int(self.binning.get()))
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
newTs.write(properties=False)
outputInterpolatedSetOfTiltSeries.update(newTs)
outputInterpolatedSetOfTiltSeries.updateDim()
outputInterpolatedSetOfTiltSeries.write()
self._store()
def generateOutputStackStep(self, tsObjId):
outputNormalizedSetOfTiltSeries = self.getOutputNormalizedSetOfTiltSeries(
)
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
tmpPrefix = self._getTmpPath(tsId)
paramsNewstack = {
'input': os.path.join(tmpPrefix,
ts.getFirstItem().parseFileName()),
'output': os.path.join(extraPrefix,
ts.getFirstItem().parseFileName()),
'bin': int(self.binning.get()),
'imagebinned': 1.0,
}
argsNewstack = "-input %(input)s " \
"-output %(output)s " \
"-bin %(bin)d " \
"-imagebinned %(imagebinned)s "
if self.floatDensities.get() != 0:
argsNewstack += " -FloatDensities " + str(
self.floatDensities.get())
if self.floatDensities.get() == 2:
if self.meanSdToggle.get() == 0:
argsNewstack += " -MeanAndStandardDeviation " + str(self.scaleMean.get()) + "," + \
str(self.scaleSd.get())
elif self.floatDensities.get() == 4:
argsNewstack += " -ScaleMinAndMax " + str(
self.scaleMax.get()) + "," + str(self.scaleMin.get())
else:
if self.scaleRangeToggle.get() == 0:
argsNewstack += " -ScaleMinAndMax " + str(self.scaleRangeMax.get()) + "," + \
str(self.scaleRangeMin.get())
if self.getModeToOutput() is not None:
argsNewstack += " -ModeToOutput " + str(self.getModeToOutput())
Plugin.runImod(self, 'newstack', argsNewstack % paramsNewstack)
newTs = tomoObj.TiltSeries(tsId=tsId)
newTs.copyInfo(ts)
outputNormalizedSetOfTiltSeries.append(newTs)
if self.binning > 1:
newTs.setSamplingRate(ts.getSamplingRate() *
int(self.binning.get()))
for index, tiltImage in enumerate(ts):
newTi = tomoObj.TiltImage()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(
index + 1,
(os.path.join(extraPrefix, tiltImage.parseFileName())))
if self.binning > 1:
newTi.setSamplingRate(tiltImage.getSamplingRate() *
int(self.binning.get()))
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
newTs.write(properties=False)
outputNormalizedSetOfTiltSeries.update(newTs)
outputNormalizedSetOfTiltSeries.updateDim()
outputNormalizedSetOfTiltSeries.write()
self._store()
def computeInterpolatedStackStep(self, tsObjId):
outputInterpolatedSetOfTiltSeries = self.getOutputInterpolatedSetOfTiltSeries(
)
ts = self.inputSetOfTiltSeries.get()[tsObjId]
tsId = ts.getTsId()
extraPrefix = self._getExtraPath(tsId)
tmpPrefix = self._getTmpPath(tsId)
paramsAlignment = {
'input':
os.path.join(tmpPrefix,
ts.getFirstItem().parseFileName()),
'output':
os.path.join(extraPrefix,
ts.getFirstItem().parseFileName()),
'xform':
os.path.join(extraPrefix,
ts.getFirstItem().parseFileName(extension=".prexg")),
'bin':
int(self.binning.get()),
'imagebinned':
1.0
}
argsAlignment = "-input %(input)s " \
"-output %(output)s " \
"-xform %(xform)s " \
"-bin %(bin)d " \
"-imagebinned %(imagebinned)s"
Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
newTs = tomoObj.TiltSeries(tsId=tsId)
newTs.copyInfo(ts)
outputInterpolatedSetOfTiltSeries.append(newTs)
if self.binning > 1:
newTs.setSamplingRate(ts.getSamplingRate() *
int(self.binning.get()))
for index, tiltImage in enumerate(ts):
newTi = tomoObj.TiltImage()
newTi.copyInfo(tiltImage, copyId=True)
newTi.setLocation(
index + 1,
(os.path.join(extraPrefix, tiltImage.parseFileName())))
if self.binning > 1:
newTi.setSamplingRate(tiltImage.getSamplingRate() *
int(self.binning.get()))
newTs.append(newTi)
ih = ImageHandler()
x, y, z, _ = ih.getDimensions(newTs.getFirstItem().getFileName())
newTs.setDim((x, y, z))
newTs.write(properties=False)
outputInterpolatedSetOfTiltSeries.update(newTs)
outputInterpolatedSetOfTiltSeries.updateDim()
outputInterpolatedSetOfTiltSeries.write()
self._store()