def _processMovie(self, movieId, movieName, movieFolder):
"""call program here"""
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
inMovieName = os.path.join(movieFolder, movieName)
if movieName.endswith(".mrc"):
movieNameAux = inMovieName
elif movieName.endswith(".mrcs"):
movieNameAux = pwutils.replaceExt(inMovieName, "mrc")
createLink(inMovieName, movieNameAux)
movieNameAux = pwutils.replaceExt(movieName, "mrc")
else:
micFnMrc = pwutils.replaceExt(inMovieName, "mrc")
ImageHandler().convert(inMovieName, micFnMrc, DT_FLOAT)
movieNameAux = pwutils.replaceExt(movieName, "mrc")
# get number of frames
if self.alignFrameRange == -1:
numberOfFramesPerMovie = self.inputMovies.get().getDimensions()[3]
else:
numberOfFramesPerMovie = self.alignFrameRange.get()
doApplyDoseFilter = self.doApplyDoseFilter.get()
exposurePerFrame = self.exposurePerFrame.get()
self._argsUnblur(
movieNameAux, movieFolder, movieId, numberOfFramesPerMovie, doApplyDoseFilter, exposurePerFrame
)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _visualize(self, obj, **args):
cls = type(obj)
if issubclass(cls, Volume) or issubclass(cls, Image):
fn = obj.getFileName()
if pwutils.getExt(fn) in ['.stk', '.vol', '.xmp']:
# eman2 expects spider files with .spi extension only
pwutils.createLink(fn, pwutils.replaceExt(fn, 'spi'))
fn = pwutils.replaceExt(fn, 'spi')
view = CommandView("%s %s &" %
(Plugin.getProgram('e2display.py'), fn),
env=Plugin.getEnviron())
return [view]
elif isinstance(obj, EmanProtBoxing):
coords = obj.getCoords()
if coords:
return DataViewer._visualize(self, obj.outputCoordinates)
elif isinstance(obj, EmanProtInitModel):
obj = obj.outputVolumes
fn = obj.getFileName()
labels = 'id enabled comment _filename '
objCommands = "'%s' '%s' '%s'" % (OBJCMD_CLASSAVG_PROJS,
OBJCMD_PROJS, OBJCMD_INITVOL)
self._views.append(
ObjectView(self._project,
obj.strId(),
fn,
viewParams={
showj.MODE: showj.MODE_MD,
showj.VISIBLE: labels,
showj.RENDER: '_filename',
showj.OBJCMDS: objCommands
}))
return self._views
elif isinstance(obj, EmanProtInitModelSGD):
obj = obj.outputVolumes
fn = obj.getFileName()
labels = 'id enabled comment _filename '
objCommands = "'%s'" % OBJCMD_CLASSAVG_PROJS
self._views.append(
ObjectView(self._project,
obj.strId(),
fn,
viewParams={
showj.MODE: showj.MODE_MD,
showj.VISIBLE: labels,
showj.RENDER: '_filename',
showj.OBJCMDS: objCommands
}))
return self._views
def getFinalCorrectionImagePath(self, correctionImage):
""" Returns the final path to the correction image (converted or not)
or and exception correctionImage does not exists"""
# Return if the correctionImage is None or the empty string
if not correctionImage:
return None
elif not os.path.exists(correctionImage):
raise Exception(
"Correction image is set but not present in the "
"file system. Either clean the value in the import "
"protocol or copy the file to %s." % correctionImage)
# Do we need to convert?
convertTo = self._getConvertExtension(correctionImage)
if convertTo is not None:
# Get the base name
fileName = basename(correctionImage)
# Replace extension
fileName = pwutils.replaceExt(fileName, convertTo)
# "Place" it at extra folder.
return self._getExtraPath(fileName)
else:
return correctionImage
def _showOneColorslice(self, param=None):
if (exists(self.protocol._getExtraPath("mgresolution.mrc"))):
imageFile = self.protocol._getExtraPath("mgresolution.mrc")
else:
imageFile = self.protocol._getExtraPath(OUTPUT_RESOLUTION_FILE)
if not os.path.exists(imageFile):
imageFile = replaceExt(imageFile, 'mrc')
imgData, min_Res, max_Res, voldim = self.getImgData(imageFile)
xplotter = XmippPlotter(x=1, y=1, mainTitle="Local Resolution Slices "
"along %s-axis."
% self._getAxis())
sliceNumber = self.sliceNumber.get()
if sliceNumber < 0:
sliceNumber = int(voldim[0] / 2)
else:
sliceNumber -= 1
# sliceNumber has no sense to start in zero
a = xplotter.createSubPlot("Slice %s" % (sliceNumber + 1), '', '')
matrix = self.getSliceImage(imgData, sliceNumber, self._getAxis())
plot = xplotter.plotMatrix(a, matrix, self.lowest.get(), self.highest.get(),
cmap=self.getColorMap(),
interpolation="nearest")
xplotter.getColorBar(plot)
return [xplotter]
def __convertCorrectionImage(self, correctionImage):
""" Will convert a gain or dark file to a compatible one and return
the final file name. Otherwise, will return same passed parameter"""
# Get final correction image file
finalName = self.getFinalCorrectionImagePath(correctionImage)
# If correctionImage is None, finalName will be None
if finalName is None:
return None
elif not os.path.exists(finalName):
# Conversion never happened...
print('converting %s to %s' % (correctionImage, finalName))
if correctionImage.endswith('.gain'): # treat as tif file
fnBase = basename(correctionImage)
tmpLink = self._getTmpPath(pwutils.replaceExt(fnBase, 'tif'))
pwutils.createAbsLink(correctionImage, tmpLink)
correctionImage = tmpLink
emlib.image.ImageHandler().convert(correctionImage, finalName)
# return final name
return os.path.abspath(finalName)
def appendMicsFromTomogram(self, output, tomo):
self.info("Creating micrographs for %s" % tomo.getFileName())
# Load the tomogram
ih = ImageHandler()
img = ih.read(tomo.getFileName())
data = img.getData()
# For each slice
for index in range(0, len(data), self.slicesGap.get()):
self.debug("Creating micrograph for slice %s" % index)
micName = tomoSliceToMicName(tomo, index)
outputMicName = self._getExtraPath(micName)
outputMicName = replaceExt(outputMicName, "mrc")
slice = data[index]
micImg = ImageHandler()
micImg._img.setData(slice)
micImg.write(micImg._img, outputMicName)
# Create the micrograph metadata object
newMic = Micrograph()
newMic.setFileName(outputMicName)
newMic.setMicName(micName)
newMic.setSamplingRate(tomo.getSamplingRate())
# Append it
output.append(newMic)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
# Write dummy auxiliary shift file.
# TODO: this should be done properly when we define
# how to transfer between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName,'w')
shift = ("0 " * numberOfFrames + "\n" ) * 2
f.write(shift)
f.close()
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n"%movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
# Write dummy auxiliary shift file.
# TODO: this should be done properly when we define
# how to transfer between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName, 'w')
shift = ("0 " * numberOfFrames + "\n") * 2
f.write(shift)
f.close()
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsSummovie(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
"""call program here"""
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
inMovieName = os.path.join(movieFolder, movieName)
if movieName.endswith(".mrc"):
movieNameAux = inMovieName
elif movieName.endswith(".mrcs"):
movieNameAux = pwutils.replaceExt(inMovieName, "mrc")
createLink(inMovieName, movieNameAux)
movieNameAux = pwutils.replaceExt(movieName, "mrc")
else:
micFnMrc = pwutils.replaceExt(inMovieName, "mrc")
ImageHandler().convert(inMovieName, micFnMrc, DT_FLOAT)
movieNameAux = pwutils.replaceExt(movieName, "mrc")
# get number of frames
if self.alignFrameRange == -1:
numberOfFramesPerMovie = self.inputMovies.get().getDimensions()[3]
else:
numberOfFramesPerMovie = self.alignFrameRange.get()
# write dummy auxiliary shift file.
# TODO: this should be done properly when we define how to transfer shift
# between movies
shiftFnName = os.path.join(movieFolder, self._getShiftFnName(movieId))
f = open(shiftFnName, "w")
shift = ("0 " * numberOfFramesPerMovie + "\n") * 2
f.write(shift)
f.close()
doApplyDoseFilter = self.doApplyDoseFilter.get()
exposurePerFrame = self.exposurePerFrame.get()
self._argsSummovie(
movieNameAux, movieFolder, movieId, numberOfFramesPerMovie, doApplyDoseFilter, exposurePerFrame
)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _getOutputMicrograph(self, mic):
""" Return the name of the output micrograph, given
the input Micrograph object.
"""
fn = mic.getFileName()
extFn = getExt(fn)
if extFn != ".mrc":
fn = replaceExt(fn, "mrc")
fnOut = self._getExtraPath(basename(fn))
return fnOut
def _getOutputMicrograph(self, mic):
""" Return the name of the output micrograph, given
the input Micrograph object.
"""
fn = mic.getFileName()
extFn = getExt(fn)
if extFn != ".mrc":
fn = replaceExt(fn, "mrc")
fnOut = self._getExtraPath(basename(fn))
return fnOut
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _processMovie(self, movieId, movieName, movieFolder):
# if not mrc convert format to mrc
# special case is mrc but ends in mrcs
moviePath = os.path.join(movieFolder, movieName)
movieNameMrc = pwutils.replaceExt(movieName, "mrc")
moviePathMrc = pwutils.replaceExt(moviePath, "mrc")
ih = ImageHandler()
if movieName.endswith('.mrc'):
pass # Do nothing
elif movieName.endswith('.mrcs'):
# Create a link to the mrc or mrcs file but with .mrc extension
createLink(moviePath, moviePathMrc)
else:
# Convert to mrc if the movie is in other format
ih.convert(moviePath, moviePathMrc, DT_FLOAT)
_, _, z, n = ih.getDimensions(moviePathMrc)
numberOfFrames = max(z, n) # Deal with mrc ambiguity
if self.alignFrameRange != -1:
if self.alignFrameRange > numberOfFrames:
raise Exception('Frame number (%d) is greater than '
'the total frames of the movie (%d)' %
(numberOfFrames, self.alignFrameRange))
numberOfFrames = self.alignFrameRange.get()
self._argsUnblur(movieNameMrc, movieFolder, movieId, numberOfFrames)
try:
self.runJob(self._program, self._args, cwd=movieFolder)
except:
print("ERROR: Movie %s failed\n" % movieName)
logFile = self._getLogFile(movieId)
def _pickMicrographStep(self, mics, args):
""" Main func that runs the picking job.
:param mics: micrograph list
:param args: programs args
"""
for mic in mics:
micName = mic.getFileName()
outMic = os.path.join(self._getTmpPath(),
pwutils.replaceBaseExt(micName, 'mrc'))
ctf = self.ctfDict[mic.getMicName()]
args.update({'micName': outMic,
'logFn': self._getLogFn(mic),
'outStack': self._getStackFn(mic),
'phaseShift': ctf.getPhaseShift() or 0.0,
'defocusU': ctf.getDefocusU(),
'defocusV': ctf.getDefocusV(),
'defocusAngle': ctf.getDefocusAngle()
})
if self.pickType == 1:
args.update({
'refsFn': self._getExtraPath('references.mrc'),
'useRadAvg': 'YES' if self.useRadAvg else 'NO',
'rotateRef': self.rotateRef.get(),
})
argsStr = self._getArgsStr()
cmdArgs = argsStr % args
try:
self.runJob(self._getProgram(), cmdArgs,
env=Plugin.getEnviron())
# Move output coords from tmp to extra
pltFn = pwutils.replaceExt(self._getStackFn(mic), 'plt')
pwutils.moveFile(pltFn, self._getPltFn(mic))
# Clean tmp folder
pwutils.cleanPath(outMic)
pwutils.cleanPath(self._getLogFn(mic))
pwutils.cleanPath(self._getStackFn(mic))
except Exception as e:
self.error("ERROR: Picking has failed for %s. %s" % (
outMic, self._getErrorFromPickerTxt(mic, e)))
def _showVolumeColorSlices(self, param=None):
if (exists(self.protocol._getExtraPath("mgresolution.mrc"))):
imageFile = self.protocol._getExtraPath("mgresolution.mrc")
else:
imageFile = self.protocol._getExtraPath(OUTPUT_RESOLUTION_FILE)
if not os.path.exists(imageFile):
imageFile = replaceExt(imageFile, 'mrc')
imgData, min_Res, max_Res, voldim = self.getImgData(imageFile)
xplotter = XmippPlotter(x=2, y=2, mainTitle="Local Resolution Slices "
"along %s-axis."
% self._getAxis())
# The slices to be shown are close to the center. Volume size is divided in
# 9 segments, the fouth central ones are selected i.e. 3,4,5,6
for i in range(3, 7):
sliceNumber = self.getSlice(i, imgData)
a = xplotter.createSubPlot("Slice %s" % (sliceNumber + 1), '', '')
matrix = self.getSliceImage(imgData, sliceNumber, self._getAxis())
plot = xplotter.plotMatrix(a, matrix, self.lowest.get(), self.highest.get(),
cmap=self.getColorMap(),
interpolation="nearest")
xplotter.getColorBar(plot)
return [xplotter]
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
given the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print("\n")
print("*" * 80)
print("* Launching test: ", fileKey)
print("*" * 80)
is2D = alignType == ALIGN_2D
if fileKey == 'alignShiftRotExp':
# relion requires mrcs stacks
origFn = self.dataset.getFile(fileKey)
stackFn = replaceExt(origFn, ".mrcs")
createLink(origFn, stackFn)
else:
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile("reconstruction/gold/" + fileKey +
'_Gold_rln_output.vol')
if PRINT_FILES:
print("BINARY DATA: ", stackFn)
print("SET1: ", partFn1)
print(" MD: ", mdFn)
print("SET2: ", partFn2)
print("OUTPUT: ", outputFn)
print("GOLD: ", goldFn)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
acq = Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000)
og = OpticsGroups.create(rlnMtfFileName="mtfFile1.star",
rlnImageSize=128)
partSet.setSamplingRate(1.0)
partSet.setAcquisition(acq)
og.toImages(partSet)
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [np.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print("Partset", partFn1)
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
starWriter = convert.createWriter()
starWriter.writeSetOfParticles(partSet, mdFn, alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
convert.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
convert.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
convert.readSetOfParticles(mdFn,
partSet2,
rowToFunc=convert.rowToVolume,
alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print("-" * 5)
print(img.getFileName(), img.getIndex())
print('m1:\n', m1, convert.geometryFromMatrix(m1, False))
print('m2:\n', m2, convert.geometryFromMatrix(m2, False))
self.assertTrue(np.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
if CLEAN_IMAGES:
cleanPath(outputFn)
def _getMovieFn(self, movie):
movieFn = movie.getFileName()
if movieFn.endswith("tiff"):
return pwutils.replaceExt(movieFn, "tif")
else:
return movieFn
def _getOutputVolFn(self, fn):
return replaceExt(fn, '_origSize.mrc')
def _getMovieFn(self, movie):
movieFn = movie.getFileName()
if movieFn.endswith("mrcs"):
return pwutils.replaceExt(movieFn, self.CONVERT_TO_MRC)
else:
return movieFn
def generate(self, finished):
project = self.protocol.getProject()
# Project Properties Section
# Do not delete this variables. We are using them
# in an eval command
self.projectName = project.getShortName()
startTime = pwutils.dateStr(project.getCreationTime(), secs=True),
tnow = datetime.now()
_now = project.getCreationTime()
dateStr = pwutils.prettyTime(dt=tnow, secs=True),
projectDuration = pwutils.prettyDelta(tnow -
project.getCreationTime()),
projectStatus = "FINISHED" if finished else "RUNNING",
scipionVersion = os.environ['SCIPION_VERSION'],
# create structure with information related with the properties section
# the primmary key is time, projectionName, section
# if we insert two points with these three values idential
# the second updates the first
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'properties'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "properties")
fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2}
fieldNames = {
'dateStr': "<b>Last Update</b>",
'projectDuration': '<b>Duration</b>',
'projectStatus': '<b>Status</b>'
}
if firstTime:
fieldKeys['startTime'] = 5
fieldKeys['scipionVersion'] = 1
fieldNames['startTime'] = '<b>Start Time</b>'
fieldNames['scipionVersion'] = '<b>Scipion Version</b>'
for metric, delta in fieldKeys.items():
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = fieldNames[metric]
# str is need because all values must have the same type
fields['valueStr'] = str(eval(metric)[0])
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
self.confParser.set("project", "properties", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# acquisition section
self.provider.refreshObjects()
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'acquisition'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "acquisition")
if firstTime:
delta = 0
for metricName, value in self.provider.acquisition:
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = "<b>" + metricName + "</b>"
fields['valueNum'] = float(value)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
delta = delta - 1
# update first time only if some date has been read.
# do not place this upside the loop
self.confParser.set("project", "acquisition", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# send summary section
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'summary'
pointsDict['tags'] = tags
localNow = _now
for obj in self.provider.getObjects():
fields = {}
localNow = localNow + timedelta(seconds=-1)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
fields['protocolName'] = '<b>' + obj.name + '</b>'
fields['output'] = ""
fields['size'] = ""
else:
fields['protocolName'] = ""
fields['output'] = obj.output
fields['size'] = str(obj.outSize)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
# Ctf monitor chart data
last_id = self.confParser.getint("ctf", "lastId")
listDictionaryCTF = {} if self.ctfMonitor is None else \
self.ctfMonitor.getData(lastId=last_id)
if listDictionaryCTF:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'ctf'
for ctf in listDictionaryCTF:
fields = {}
for key in ctf:
if key == 'timestamp':
pointsDict['time'] = ctf['timestamp']
# id must be a tag since tw CTF may have the same timestamp
# and time + tags must be unique
elif key == 'id':
tags['id'] = ctf['id']
# elif key == 'ctfID':
# fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>"
elif key == 'shiftPlotPath':
temp = os.path.join(self.apacheImgDir,
self.projectName,
basename(ctf[key]))
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, basename(ctf[key]))
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
elif key == 'psdPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
elif key == 'micPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
else:
fields[key] = ctf[key]
# while be use to control image creation
fields["transferImage"] = False
pointsDict['fields'] = fields
pointsDict['tags'] = tags
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("ctf", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# GAIN Section
last_id = self.confParser.getint("gain", "lastId")
listDictionaryGain = {} if self.movieGainMonitor is None else \
self.movieGainMonitor.getData(lastId=last_id)
if listDictionaryGain:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'gain'
pointsDict['tags'] = tags
counter = 1
tnow = datetime.now()
for gain in listDictionaryGain:
fields = {}
for key in gain:
fields[key] = gain[key]
pointsDict['fields'] = fields
# gain has no time information so I just
# put current time
localNow = tnow + timedelta(seconds=counter)
pointsDict[
'time'] = localNow # .strftime('%Y-%m-%dT%H:%M:%SZ')
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("gain", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# SYSTEM data
last_id = self.confParser.getint("system", "lastId")
listDictionarySystem = {} if self.sysMonitor is None else \
self.sysMonitor.getData(lastId=last_id)
if listDictionarySystem:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'system'
pointsDict['tags'] = tags
for system in listDictionarySystem:
fields = {}
for key in system:
if key == 'timestamp':
pointsDict['time'] = system['timestamp']
elif key == 'id':
continue
else:
fields[key] = system[key]
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("system", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
self.transferFiles()
return last_id # reportFinished
def getFinalMaskFileName(inMask):
if getExt(inMask.getFileName()) in COMP_EXT_MASK_LIST:
return abspath(inMask.getFileName())
else:
return abspath(replaceExt(inMask.getFileName(), '.mrc'))
def getThumbnailFn(self, inputFn):
""" Returns the default name for a thumbnail image"""
return pwutils.replaceExt(inputFn, "thumb.png")
def getThumbnailFn(inputFn):
"""Replace the extension in inputFn with thumb.png"""
return pwutils.replaceExt(inputFn, "thumb.png")
def processMovieStep(self, movieDict, hasAlignment):
movie = emobj.Movie()
movie.setAcquisition(emobj.Acquisition())
if hasAlignment:
movie.setAlignment(emobj.MovieAlignment())
movie.setAttributesFromDict(movieDict,
setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
movieDoneFn = self._getMovieDone(movie)
if self.isContinued() and os.path.exists(movieDoneFn):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(movieDoneFn)
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2',
'-d -f %s' % movieName,
cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = emlib.image.ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin,
line.strip())
ih.convert(
inputFrame,
(i + 1, os.path.join(movieFolder,
newMovieName)))
else:
newMovieName = movieName
convertExt = self._getConvertExtension(newMovieName)
correctGain = self._doCorrectGain()
if convertExt or correctGain:
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
if convertExt:
newMovieName = pwutils.replaceExt(newMovieName, convertExt)
else:
newMovieName = '%s_corrected.%s' % os.path.splitext(
newMovieName)
outputMovieFn = os.path.join(movieFolder, newMovieName)
# If the protocols wants Scipion to apply the gain, then
# there is no reason to convert, since we can produce the
# output in the format expected by the program. In some cases,
# the alignment programs can directly deal with gain and dark
# correction images, so we don't need to apply it
if self._doCorrectGain():
self.info("Correcting gain and dark '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
gain, dark = self.getGainAndDark()
self.correctGain(inputMovieFn,
outputMovieFn,
gainFn=gain,
darkFn=dark)
else:
self.info("Converting movie '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
emlib.image.ImageHandler().convertStack(
inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if self._doMovieFolderCleanUp():
self._cleanMovieFolder(movieFolder)
# Mark this movie as finished
open(movieDoneFn, 'w').close()
def processMovieStep(self, movieDict, hasAlignment):
movie = Movie()
movie.setAcquisition(Acquisition())
if hasAlignment:
movie.setAlignment(MovieAlignment())
movie.setAttributesFromDict(movieDict,
setBasic=True,
ignoreMissing=True)
movieFolder = self._getOutputMovieFolder(movie)
movieFn = movie.getFileName()
movieName = basename(movieFn)
if (self.isContinued() and os.path.exists(self._getMovieDone(movie))):
self.info("Skipping movie: %s, seems to be done" % movieFn)
return
# Clean old finished files
pwutils.cleanPath(self._getMovieDone(movie))
if self._filterMovie(movie):
pwutils.makePath(movieFolder)
pwutils.createLink(movieFn, join(movieFolder, movieName))
if movieName.endswith('bz2'):
newMovieName = movieName.replace('.bz2', '')
# We assume that if compressed the name ends with .mrc.bz2
if not exists(newMovieName):
self.runJob('bzip2',
'-d -f %s' % movieName,
cwd=movieFolder)
elif movieName.endswith('tbz'):
newMovieName = movieName.replace('.tbz', '.mrc')
# We assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tar', 'jxf %s' % movieName, cwd=movieFolder)
elif movieName.endswith('.tif'):
#FIXME: It seems that we have some flip problem with compressed
# tif files, we need to check that
newMovieName = movieName.replace('.tif', '.mrc')
# we assume that if compressed the name ends with .tbz
if not exists(newMovieName):
self.runJob('tif2mrc',
'%s %s' % (movieName, newMovieName),
cwd=movieFolder)
elif movieName.endswith('.txt'):
# Support a list of frame as a simple .txt file containing
# all the frames in a raw list, we could use a xmd as well,
# but a plain text was choose to simply its generation
movieTxt = os.path.join(movieFolder, movieName)
with open(movieTxt) as f:
movieOrigin = os.path.basename(os.readlink(movieFn))
newMovieName = movieName.replace('.txt', '.mrcs')
ih = ImageHandler()
for i, line in enumerate(f):
if line.strip():
inputFrame = os.path.join(movieOrigin,
line.strip())
ih.convert(
inputFrame,
(i + 1, os.path.join(movieFolder,
newMovieName)))
else:
newMovieName = movieName
if (self.CONVERT_TO_MRC
and not (newMovieName.endswith("mrc")
or newMovieName.endswith("mrcs"))):
inputMovieFn = os.path.join(movieFolder, newMovieName)
if inputMovieFn.endswith('.em'):
inputMovieFn += ":ems"
newMovieName = pwutils.replaceExt(newMovieName,
self.CONVERT_TO_MRC)
outputMovieFn = os.path.join(movieFolder, newMovieName)
self.info("Converting movie '%s' -> '%s'" %
(inputMovieFn, outputMovieFn))
ImageHandler().convertStack(inputMovieFn, outputMovieFn)
# Just store the original name in case it is needed in _processMovie
movie._originalFileName = pwobj.String(objDoStore=False)
movie._originalFileName.set(movie.getFileName())
# Now set the new filename (either linked or converted)
movie.setFileName(os.path.join(movieFolder, newMovieName))
self.info("Processing movie: %s" % movie.getFileName())
self._processMovie(movie)
if pwutils.envVarOn('SCIPION_DEBUG_NOCLEAN'):
self.info('Clean movie data DISABLED. '
'Movie folder will remain in disk!!!')
else:
self.info("Erasing.....movieFolder: %s" % movieFolder)
os.system('rm -rf %s' % movieFolder)
# cleanPath(movieFolder)
# Mark this movie as finished
open(self._getMovieDone(movie), 'w').close()
def _getMicDone(self, mic):
fn = mic.getFileName()
extFn = getExt(fn)
if extFn != ".mrc":
fn = replaceExt(fn, "mrc")
return self._getExtraPath('%s' % basename(fn))
def getThumbnailFn(inputFn):
"""Replace the extension in inputFn with thumb.png"""
return pwutils.replaceExt(inputFn, "thumb.png")
def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='png', micIdSet=None):
"""Adds to self.thumbPaths the paths to the report thumbnails
that come from the alignment and/or ctf protocol.
===== Params =====
- thumbsDone: how many thumbnails have already been generated.
we will get paths starting from this index
- ctfData: dict resulting from ctfMonitor.getData()
- ext: extension of the thumbnail images. Defaults to png.
- micIdSet: mic indexes to use
"""
# get psd thumbs from ctfData
if ctfData is not None:
for i in range(thumbsDone, len(ctfData[PSD_PATH])):
psdPath = ctfData[PSD_PATH][i]
movie = basename(os.path.dirname(psdPath))
psdThumb = join(PSD_THUMBS, "%s_%s" % (movie, pwutils.replaceExt(basename(psdPath), ext)))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
# get alignment and mic thumbs
if self.alignProtocol is not None:
getMicFromCTF = False
updatedProt = getUpdatedProtocol(self.alignProtocol)
if hasattr(updatedProt, 'outputMicrographs'):
outputSet = updatedProt.outputMicrographs
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
elif self.ctfProtocol is not None:
getMicFromCTF = True
updatedProt = getUpdatedProtocol(self.ctfProtocol)
if hasattr(updatedProt, 'outputCTF'):
outputSet = updatedProt.outputCTF
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
else:
return
for micId in micIdSet[thumbsDone:]:
mic = outputSet[micId]
if getMicFromCTF:
mic = mic.getMicrograph()
if hasattr(mic, 'thumbnail'):
srcMicFn = abspath(mic.thumbnail.getFileName())
else:
srcMicFn = abspath(mic.getFileName())
micThumbFn = join(MIC_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext))
self.thumbPaths[MIC_PATH].append(srcMicFn)
self.thumbPaths[MIC_THUMBS].append(micThumbFn)
shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None))
if shiftPlot is not None:
shiftPath = "" if shiftPlot is None else abspath(shiftPlot.getFileName())
shiftCopy = "" if shiftPlot is None else join(SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext))
self.thumbPaths[SHIFT_PATH].append(shiftPath)
self.thumbPaths[SHIFT_THUMBS].append(shiftCopy)
else:
if SHIFT_PATH in self.thumbPaths:
self.thumbPaths.pop(SHIFT_PATH, None)
if SHIFT_THUMBS in self.thumbPaths:
self.thumbPaths.pop(SHIFT_THUMBS, None)
self.thumbPaths[MIC_ID].append(micId)
if self.ctfProtocol is None:
def getMicPSDPath(mic):
if hasattr(mic, 'psdJpeg'):
return mic.psdJpeg.getFileName()
elif hasattr(mic,'psdCorr'):
return mic.psdCorr.getFileName()
else:
return None
psdPath = getMicPSDPath(mic)
psdThumb = None
if psdPath ==None:
psdThumb = join(PSD_THUMBS, pwutils.replaceExt(basename(psdPath), ext))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
else:
if PSD_THUMBS in self.thumbPaths:
self.thumbPaths.pop(PSD_THUMBS, None)
if PSD_PATH in self.thumbPaths:
self.thumbPaths.pop(PSD_PATH, None)
def getThumbPaths(self,
thumbsDone=0,
ctfData=None,
ext='png',
micIdSet=None):
"""Adds to self.thumbPaths the paths to the report thumbnails
that come from the alignment and/or ctf protocol.
===== Params =====
- thumbsDone: how many thumbnails have already been generated.
we will get paths starting from this index
- ctfData: dict resulting from ctfMonitor.getData()
- ext: extension of the thumbnail images. Defaults to png.
- micIdSet: mic indexes to use
"""
# get psd thumbs from ctfData
if ctfData is not None:
for i in range(thumbsDone, len(ctfData[PSD_PATH])):
psdPath = ctfData[PSD_PATH][i]
movie = basename(os.path.dirname(psdPath))
psdThumb = join(
PSD_THUMBS, "%s_%s" %
(movie, pwutils.replaceExt(basename(psdPath), ext)))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
# get alignment and mic thumbs
if self.alignProtocol is not None:
getMicFromCTF = False
updatedProt = getUpdatedProtocol(self.alignProtocol)
if hasattr(updatedProt, 'outputMicrographs'):
outputSet = updatedProt.outputMicrographs
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
elif self.ctfProtocol is not None:
getMicFromCTF = True
updatedProt = getUpdatedProtocol(self.ctfProtocol)
if hasattr(updatedProt, 'outputCTF'):
outputSet = updatedProt.outputCTF
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
else:
return
for micId in micIdSet[thumbsDone:]:
mic = outputSet[micId]
if getMicFromCTF:
mic = mic.getMicrograph()
if hasattr(mic, 'thumbnail'):
srcMicFn = abspath(mic.thumbnail.getFileName())
else:
srcMicFn = abspath(mic.getFileName())
micThumbFn = join(MIC_THUMBS,
pwutils.replaceExt(basename(srcMicFn), ext))
self.thumbPaths[MIC_PATH].append(srcMicFn)
self.thumbPaths[MIC_THUMBS].append(micThumbFn)
shiftPlot = (getattr(mic, 'plotCart', None)
or getattr(mic, 'plotGlobal', None))
if shiftPlot is not None:
shiftPath = "" if shiftPlot is None else abspath(
shiftPlot.getFileName())
shiftCopy = "" if shiftPlot is None else join(
SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext))
self.thumbPaths[SHIFT_PATH].append(shiftPath)
self.thumbPaths[SHIFT_THUMBS].append(shiftCopy)
else:
if SHIFT_PATH in self.thumbPaths:
self.thumbPaths.pop(SHIFT_PATH, None)
if SHIFT_THUMBS in self.thumbPaths:
self.thumbPaths.pop(SHIFT_THUMBS, None)
self.thumbPaths[MIC_ID].append(micId)
if self.ctfProtocol is None:
psdPath = mic.psdJpeg.getFileName() if hasattr(
mic, 'psdJpeg') else mic.psdCorr.getFileName()
psdThumb = join(PSD_THUMBS,
pwutils.replaceExt(basename(psdPath), ext))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
def _getMovieFn(self, movie):
movieFn = movie.getFileName()
if movieFn.endswith("mrcs"):
return pwutils.replaceExt(movieFn, self.CONVERT_TO_MRC)
else:
return movieFn
