def buildChildren(self, child_, nodeName_): if child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'b': obj_ = XSDataBoolean() obj_.build(child_) self.setB(obj_) XSData.buildChildren(self, child_, nodeName_)
def buildChildren(self, child_, nodeName_): if child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'flush': obj_ = XSDataBoolean() obj_.build(child_) self.setFlush(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'item': obj_ = XSDataString() obj_.build(child_) self.item.append(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'query': obj_ = XSDataQuery() obj_.build(child_) self.query.append(obj_) XSDataInput.buildChildren(self, child_, nodeName_)
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'timedOut':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setTimedOut(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'finalSize':
obj_ = XSDataInteger()
obj_.build(child_)
self.setFinalSize(obj_)
XSDataResult.buildChildren(self, child_, nodeName_)
from XSDataCommon import XSDataBoolean
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataString
from XSDataCommon import XSDataInteger
from XSDataCommon import XSDataFloat
x = XSDataBoolean(True)
print x.value
print x.marshal()
x = XSDataBoolean(False)
print x.value
print x.marshal()
x = XSDataBoolean('true')
print x.value
print x.marshal()
x = XSDataBoolean('False')
print x.value
print x.marshal()
x = XSDataBoolean(0)
print x.value
print x.marshal()
x = XSDataBoolean(1)
print x.value
print x.marshal()
x = XSDataBoolean(2)
print x.value
print x.marshal()
y = XSDataFloat('this is not a float')
print y.value
print y.marshal()
z = XSDataInteger('this is not an integer')
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlSaxsModelingv1_1.process")
xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile,
unit=XSDataString(self.unit),
symmetry=XSDataString(
self.symmetry),
mode=XSDataString(self.mode))
for i in range(self.dammif_jobs):
dammif = self.loadPlugin(self.strPluginExecDammif)
dammif.connectSUCCESS(self.doSuccessExecDammif)
dammif.connectFAILURE(self.doFailureExecDammif)
xsd = xsDataInputDammif.copyViaDict()
xsd.order = XSDataInteger(i + 1)
dammif.dataInput = xsd
self.addPluginToActionCluster(dammif)
self.dammif_plugins.append(dammif)
self.executeActionCluster()
self.synchronizeActionCluster()
for plugin in self.dammif_plugins:
if plugin.isFailure():
self.ERROR("dammif plugin %s-%08i failed" %
(plugin.getName(), plugin.getId()))
self.setFailure()
self.retrieveMessages(plugin)
if self.isFailure():
return
# retrieve results from best dammif
self.dammif = self.bestDammif()
self.chi2plot("chi2_R.png")
self.result.chiRfactorPlot = XSDataFile(
XSDataString(os.path.join(self.getWorkingDirectory(),
"chi2_R.png")))
# temporary results: use best dammif
self.result.fitFile = self.dammif.dataOutput.fitFile
self.result.logFile = self.dammif.dataOutput.logFile
self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile
self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile
# align models, compute NSD and choose the reference model
inputfiles = [
self.dammif_plugins[idx].dataOutput.pdbMoleculeFile.path.value
for idx in range(self.dammif_jobs)
]
align = AlignModels(inputfiles, slow=False)
outputfiles = []
for i in range(self.dammif_jobs):
outputfiles.append(
os.path.join(self.getWorkingDirectory(),
"model-%02i.pdb" % (i + 1)))
align.outputfiles = outputfiles
#align.outputfiles = ["model-%02i.pdb" % (i+1) for i in range(self.dammif_jobs)]
align.validmodels = self.valid
align.assign_models()
align.makeNSDarray()
align.alignment_reference()
self.ref = align.reference
pngfile = os.path.join(self.getWorkingDirectory(), "nsd.png")
align.plotNSDarray(filename=pngfile, save=True)
self.result.nsdPlot = XSDataFile(XSDataString(pngfile))
# Now that all (valid) models are aligned we can combine them using damaver
pdbFiles = [XSDataFile(XSDataString(align.outputfiles[self.ref]))]
for idx in range(self.dammif_jobs):
if self.valid[idx] and idx != self.ref:
pdbFiles.append(
XSDataFile(XSDataString(align.outputfiles[idx])))
damaver = self.loadPlugin(self.strPluginExecDamaver)
damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles,
automatic=XSDataBoolean(False))
damaver.connectSUCCESS(self.doSuccessExecDamaver)
damaver.connectFAILURE(self.doFailureExecDamaver)
damaver.executeSynchronous()
if self.isFailure():
return
damfilt = self.loadPlugin(self.strPluginExecDamfilt)
damfilt.dataInput = XSDataInputDamfilt(
inputPdbFile=damaver.dataOutput.damaverPdbFile)
damfilt.connectSUCCESS(self.doSuccessExecDamfilt)
damfilt.connectFAILURE(self.doFailureExecDamfilt)
damfilt.execute()
########################################################################
# TODO: This is a dead end : do it in parallel
########################################################################
if self.isFailure():
return
damstart = self.loadPlugin(self.strPluginExecDamstart)
damstart.dataInput = XSDataInputDamstart(
inputPdbFile=damaver.dataOutput.damaverPdbFile)
damstart.connectSUCCESS(self.doSuccessExecDamstart)
damstart.connectFAILURE(self.doFailureExecDamstart)
damstart.executeSynchronous()
if self.isFailure():
return
########################################################################
# Finally call dammin
########################################################################
if self.config.get("do_dammin") in ["False", "0", False, 0]:
pass
#return
dammin = self.loadPlugin(self.strPluginExecDammin)
dammin.dataInput = XSDataInputDammin(
pdbInputFile=damstart.dataOutput.outputPdbFile,
gnomOutputFile=self.xsGnomFile,
symmetry=XSDataString(self.symmetry),
mode=XSDataString(self.mode))
dammin.connectSUCCESS(self.doSuccessExecDammin)
dammin.connectFAILURE(self.doFailureExecDammin)
dammin.executeSynchronous()
def testSetDataModelInput(self):
edPluginBest = self.createPlugin()
xsPluginItemGood01 = self.getPluginConfiguration(
os.path.join(self.m_pyStrDataPath, "XSConfiguration.xml"))
edPluginBest.setConfiguration(xsPluginItemGood01)
edPluginBest.setScriptExecutable("cat")
edPluginBest.configure()
from XSDataBestv1_2 import XSDataInputBest
xsDataInputBest = XSDataInputBest()
from XSDataCommon import XSDataAbsorbedDoseRate
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataString
from XSDataCommon import XSDataTime
from XSDataCommon import XSDataFile
from XSDataCommon import XSDataSpeed
from XSDataCommon import XSDataString
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataBoolean
xsDataInputBest.setCrystalAbsorbedDoseRate(
XSDataAbsorbedDoseRate(0.22E+06))
xsDataInputBest.setCrystalShape(XSDataDouble(1))
xsDataInputBest.setCrystalSusceptibility(XSDataDouble(1.5))
xsDataInputBest.setDetectorType(XSDataString("q210-2x"))
xsDataInputBest.setBeamExposureTime(XSDataTime(1))
xsDataInputBest.setBeamMaxExposureTime(XSDataTime(10000))
xsDataInputBest.setBeamMinExposureTime(XSDataTime(0.1))
xsDataInputBest.setGoniostatMinRotationWidth(XSDataAngle(0.1))
xsDataInputBest.setGoniostatMaxRotationSpeed(XSDataSpeed(10))
xsDataInputBest.setAimedResolution(XSDataDouble(2))
xsDataInputBest.setAimedRedundancy(XSDataDouble(6.5))
xsDataInputBest.setAimedCompleteness(XSDataDouble(0.9))
xsDataInputBest.setAimedIOverSigma(XSDataDouble(3))
xsDataInputBest.setComplexity(XSDataString("min"))
xsDataInputBest.setAnomalousData(XSDataBoolean(False))
fileDirectory = edPluginBest.getWorkingDirectory()
bestFileContentDat = EDUtilsFile.readFile(
os.path.join(self.m_pyStrDataPath, "bestfile.dat"))
xsDataInputBest.setBestFileContentDat(XSDataString(bestFileContentDat))
bestFileContentPar = EDUtilsFile.readFile(
os.path.join(self.m_pyStrDataPath, "bestfile.par"))
xsDataInputBest.setBestFileContentPar(XSDataString(bestFileContentPar))
bestFileContentHKL = EDUtilsFile.readFile(
os.path.join(self.m_pyStrDataPath, "bestfile1.hkl"))
xsDataInputBest.addBestFileContentHKL(XSDataString(bestFileContentHKL))
xsDataInputBest.exportToFile(self.m_edObtainedInputFile)
pyStrExpectedInput = self.readAndParseFile(self.m_edReferenceInputFile)
pyStrObtainedInput = self.readAndParseFile(self.m_edObtainedInputFile)
xsDataInputExpected = XSDataInputBest.parseString(pyStrExpectedInput)
xsDataInputObtained = XSDataInputBest.parseString(pyStrObtainedInput)
EDAssert.equal(xsDataInputExpected.marshal(),
xsDataInputObtained.marshal())
EDUtilsFile.deleteFile(self.m_edObtainedInputFile)
self.cleanUp(edPluginBest)
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
EDVerbose.DEBUG(
"EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.getDataInput().getDiffractionImage(
).getPath().getValue()
strImageFileNameExtension = os.path.splitext(
strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [
".img", ".marccd", ".mccd", ".cbf"
]:
EDVerbose.error(
"Unknown image file name extension for pyarch thumbnail generator: %s"
% strPathToDiffractionImage)
self.setFailure()
else:
# Load the waitFile plugin
xsDataInputWaitFile = XSDataInputWaitFile()
xsDataInputWaitFile.setExpectedSize(
XSDataInteger(self.iExpectedSize))
xsDataInputWaitFile.setExpectedFile(
self.getDataInput().getDiffractionImage())
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputWaitFile.setTimeOut(
self.getDataInput().getWaitForFileTimeOut())
self.edPluginWaitFile = EDPluginWaitFile()
self.edPluginWaitFile.setDataInput(xsDataInputWaitFile)
# Load the execution plugin
self.edPluginExecThumbnail = self.loadPlugin(
self.strExecThumbnailPluginName)
xsDataInputExecThumbnail = XSDataInputExecThumbnail()
xsDataInputExecThumbnail.setInputImagePath(
self.getDataInput().getDiffractionImage())
xsDataInputExecThumbnail.setLevelsInvert(XSDataBoolean(True))
xsDataInputExecThumbnail.setLevelsMin(XSDataDoubleWithUnit(0.0))
xsDataDoubleWithUnitLevelsMax = XSDataDoubleWithUnit(99.95)
xsDataDoubleWithUnitLevelsMax.setUnit(XSDataString("%"))
xsDataInputExecThumbnail.setLevelsMax(
xsDataDoubleWithUnitLevelsMax)
xsDataInputExecThumbnail.setFilterDilatation([XSDataInteger(4)])
xsDataInputExecThumbnail.setLevelsColorize(XSDataBoolean(False))
xsDataInputExecThumbnail.setThumbHeight(XSDataInteger(1024))
xsDataInputExecThumbnail.setThumbWidth(XSDataInteger(1024))
xsDataInputExecThumbnail.setKeepRatio(XSDataBoolean(False))
# Output path
strImageNameWithoutExt = os.path.basename(
os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput(
).getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
EDVerbose.error(
"Cannot write to forced output directory : %s" %
strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(
strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except BaseException, e:
EDVerbose.WARNING(
"Couldn't create the directory %s" %
strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
EDVerbose.warning("Cannot write to pyarch directory: %s" %
strOutputDirname)
strOutputDirname = tempfile.mkdtemp(
"", "EDPluginPyarchThumbnailv10_", "/tmp")
EDVerbose.warning("Writing thumbnail images to: %s" %
strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(
strOutputDirname, strImageNameWithoutExt)
self.strOutputPath = os.path.join(
self.strOutputPathWithoutExtension + ".jpeg")
xsDataInputExecThumbnail.setOutputPath(
XSDataFile(XSDataString(self.strOutputPath)))
self.edPluginExecThumbnail.setDataInput(xsDataInputExecThumbnail)
def testSetDataModelInput(self):
"""
A test for whether we can obtain the expected XML by setting a certain input for the plugin.
"""
edPluginISPyB = self.createPlugin()
xsPluginItemISPyB = self.getPluginConfiguration(
os.path.join(self.getPluginTestsDataHome(), "XSConfiguration.xml"))
edPluginISPyB.setConfiguration(xsPluginItemISPyB)
edPluginISPyB.configure()
xsDataInputISPyB = XSDataInputISPyB()
xsDataISPyBImage = XSDataISPyBImage()
xsDataISPyBImage.setFileName(XSDataString("test.img"))
xsDataISPyBImage.setFileLocation(XSDataString("/tmp"))
xsDataISPyBScreening = XSDataISPyBScreening()
# xsDataISPyBScreening.setDataCollectionId( XSDataInteger ( 1 ) )
xsDataISPyBScreening.setProgramVersion(XSDataString("EDNA Prototype"))
xsDataISPyBScreeningInput = XSDataISPyBScreeningInput()
xsDataISPyBScreeningInput.setBeamX(XSDataFloat(10.4))
xsDataISPyBScreeningInput.setBeamY(XSDataFloat(2.31))
xsDataISPyBScreeningInput.setRmsErrorLimits(XSDataFloat(0.8))
xsDataISPyBScreeningInput.setMinimumFractionIndexed(XSDataFloat(0.4))
xsDataISPyBScreeningInput.setMaximumFractionRejected(XSDataFloat(0.45))
xsDataISPyBScreeningInput.setMinimumSignalToNoise(XSDataFloat(0.56))
xsDataISPyBScreeningOutput = XSDataISPyBScreeningOutput()
xsDataISPyBScreeningOutput.setStatusDescription(
XSDataString("It's just fine."))
xsDataISPyBScreeningOutput.setMosaicity(XSDataFloat(0.25))
xsDataISPyBScreeningOutput.setBeamShiftX(XSDataFloat(0.141))
xsDataISPyBScreeningOutput.setBeamShiftY(XSDataFloat(0.156))
xsDataISPyBScreeningOutputLattice = XSDataISPyBScreeningOutputLattice()
xsDataISPyBScreeningOutputLattice.setSpaceGroup(XSDataString("P222"))
xsDataISPyBScreeningStrategy = XSDataISPyBScreeningStrategy()
xsDataISPyBScreeningStrategy.setPhiStart(XSDataFloat(0))
xsDataISPyBScreeningStrategy.setPhiEnd(XSDataFloat(20))
xsDataISPyBScreeningStrategy.setRotation(XSDataFloat(1))
xsDataISPyBScreeningStrategy.setProgram(XSDataString("EDNA"))
xsDataISPyBScreeningStrategy.setAnomalous(XSDataBoolean(1))
xsDataInputISPyB.setImage(xsDataISPyBImage)
xsDataInputISPyB.setScreening(xsDataISPyBScreening)
xsDataInputISPyB.setScreeningInput(xsDataISPyBScreeningInput)
xsDataInputISPyB.setScreeningOutput(xsDataISPyBScreeningOutput)
xsDataInputISPyB.setScreeningOutputLattice(
xsDataISPyBScreeningOutputLattice)
xsDataInputISPyB.setScreeningStrategy(xsDataISPyBScreeningStrategy)
xsDataInputISPyB.outputFile(self.m_edObtainedInputFile)
strExpectedInput = self.readAndParseFile(self.m_edReferenceInputFile)
strObtainedInput = self.readAndParseFile(self.m_edObtainedInputFile)
xsDataScreeningExpected = XSDataInputISPyB.parseString(
strExpectedInput)
xsDataScreeningObtained = XSDataInputISPyB.parseString(
strObtainedInput)
pyStrExpectedXML = xsDataScreeningExpected.marshal()
pyStrObtainedXML = xsDataScreeningObtained.marshal()
EDAssert.equal(pyStrExpectedXML, pyStrObtainedXML)
self.cleanUp(edPluginISPyB)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doNoanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(masterFilePath)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(masterFilePath)))
else:
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
if self.doAnom:
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def testSetDataModelInput(self):
"""
A test for whether we can obtain the expected XML by setting a certain input for the plugin.
"""
edPluginISPyB = self.createPlugin()
xsPluginItemISPyB = self.getPluginConfiguration(
os.path.join(self.getPluginTestsDataHome(), "XSConfiguration.xml"))
edPluginISPyB.setConfiguration(xsPluginItemISPyB)
edPluginISPyB.configure()
# Create XSDataISPyB objects
xsDataISPyBImage = XSDataISPyBImage()
xsDataISPyBImage.setFileName(XSDataString("test.img"))
xsDataISPyBImage.setFileLocation(XSDataString("/tmp"))
xsDataISPyBScreening = XSDataISPyBScreening()
xsDataISPyBScreening.setProgramVersion(XSDataString("EDNA Prototype"))
xsDataISPyBScreeningInput = XSDataISPyBScreeningInput()
xsDataISPyBScreeningInput.setBeamX(XSDataDouble(10.4))
xsDataISPyBScreeningInput.setBeamY(XSDataDouble(2.31))
xsDataISPyBScreeningInput.setRmsErrorLimits(XSDataDouble(0.8))
xsDataISPyBScreeningInput.setMinimumFractionIndexed(XSDataDouble(0.4))
xsDataISPyBScreeningInput.setMaximumFractionRejected(
XSDataDouble(0.45))
xsDataISPyBScreeningInput.setMinimumSignalToNoise(XSDataDouble(0.56))
xsDataISPyBScreeningOutput = XSDataISPyBScreeningOutput()
xsDataISPyBScreeningOutput.setStatusDescription(
XSDataString("It's just fine."))
xsDataISPyBScreeningOutput.setMosaicity(XSDataDouble(0.25))
xsDataISPyBScreeningOutput.setBeamShiftX(XSDataDouble(0.141))
xsDataISPyBScreeningOutput.setBeamShiftY(XSDataDouble(0.156))
xsDataISPyBScreeningOutputLattice = XSDataISPyBScreeningOutputLattice()
xsDataISPyBScreeningOutputLattice.setSpaceGroup(XSDataString("P222"))
xsDataISPyBScreeningStrategy = XSDataISPyBScreeningStrategy()
xsDataISPyBScreeningStrategy.setPhiStart(XSDataDouble(0))
xsDataISPyBScreeningStrategy.setPhiEnd(XSDataDouble(20))
xsDataISPyBScreeningStrategy.setRotation(XSDataDouble(1))
xsDataISPyBScreeningStrategy.setProgram(XSDataString("EDNA"))
xsDataISPyBScreeningStrategy.setAnomalous(XSDataBoolean(1))
xsDataISPyBScreeningStrategyWedge = XSDataISPyBScreeningStrategyWedge()
xsDataISPyBScreeningStrategyWedge.setWedgeNumber(XSDataInteger(1))
xsDataISPyBScreeningStrategyWedge.setResolution(XSDataDouble(2.1))
xsDataISPyBScreeningStrategyWedge.setCompleteness(XSDataDouble(90))
xsDataISPyBScreeningStrategyWedge.setMultiplicity(XSDataDouble(1.5))
xsDataISPyBScreeningStrategyWedge.setDoseTotal(XSDataDouble(40.5))
xsDataISPyBScreeningStrategyWedge.setNumberOfImages(XSDataInteger(130))
xsDataISPyBScreeningStrategySubWedge = XSDataISPyBScreeningStrategySubWedge(
)
xsDataISPyBScreeningStrategySubWedge.setSubWedgeNumber(
XSDataInteger(1))
xsDataISPyBScreeningStrategySubWedge.setRotationAxis(
XSDataString("Omega"))
xsDataISPyBScreeningStrategySubWedge.setAxisStart(XSDataDouble(0.0))
xsDataISPyBScreeningStrategySubWedge.setAxisEnd(XSDataDouble(90.0))
xsDataISPyBScreeningStrategySubWedge.setExposureTime(XSDataDouble(0.5))
xsDataISPyBScreeningStrategySubWedge.setTransmission(
XSDataDouble(100.0))
xsDataISPyBScreeningStrategySubWedge.setNumberOfImages(
XSDataInteger(130))
xsDataISPyBScreeningStrategyWedgeContainer = XSDataISPyBScreeningStrategyWedgeContainer(
)
xsDataISPyBScreeningStrategyWedgeContainer.setScreeningStrategyWedge(
xsDataISPyBScreeningStrategyWedge)
xsDataISPyBScreeningStrategyWedgeContainer.getScreeningStrategySubWedge(
).append(xsDataISPyBScreeningStrategySubWedge)
xsDataISPyBScreeningStrategyContainer = XSDataISPyBScreeningStrategyContainer(
)
xsDataISPyBScreeningStrategyContainer.setScreeningStrategy(
xsDataISPyBScreeningStrategy)
xsDataISPyBScreeningStrategyContainer.getScreeningStrategyWedgeContainer(
).append(xsDataISPyBScreeningStrategyWedgeContainer)
xsDataISPyBScreeningOutputContainer = XSDataISPyBScreeningOutputContainer(
)
xsDataISPyBScreeningOutputContainer.setScreeningOutput(
xsDataISPyBScreeningOutput)
xsDataISPyBScreeningOutputContainer.getScreeningOutputLattice().append(
xsDataISPyBScreeningOutputLattice)
xsDataISPyBScreeningOutputContainer.getScreeningStrategyContainer(
).append(xsDataISPyBScreeningStrategyContainer)
xsDataISPyBScreeningRank = XSDataISPyBScreeningRank()
xsDataISPyBScreeningRank.setRankValue(XSDataDouble(1.4))
xsDataISPyBScreeningRank.setRankInformation(
XSDataString("This is the only one"))
xsDataISPyBScreeningRankSet = XSDataISPyBScreeningRankSet()
xsDataISPyBScreeningRankSet.setRankEngine(XSDataString("ISPyB"))
xsDataISPyBScreeningFile = XSDataISPyBScreeningFile()
xsDataISPyBScreeningFile.setFileType(XSDataString("log"))
xsDataISPyBScreeningFile.setDescription(
XSDataString("Output log file"))
# Write XSDataISPyB objects to files
xsDataISPyBScreening.outputFile(self.__strObtainedScreening)
xsDataISPyBScreeningInput.outputFile(self.__strObtainedScreeningInput)
xsDataISPyBScreeningOutputContainer.outputFile(
self.__strObtainedScreeningOutputContainer)
xsDataISPyBScreeningRank.outputFile(self.__strObtainedScreeningRank)
xsDataISPyBScreeningRankSet.outputFile(
self.__strObtainedScreeningRankSet)
xsDataISPyBScreeningFile.outputFile(self.__strObtainedScreeningFile)
# Compare screening
strInputScreeningExpected = self.readAndParseFile(
self.__strReferenceInputScreening)
xsDataScreeningExpected = XSDataInputISPyB.parseString(
strInputScreeningExpected)
strXMLScreeningExpected = xsDataScreeningExpected.marshal()
strScreeningObtained = self.readAndParseFile(
self.__strObtainedScreening)
xsDataScreeningObtained = XSDataInputISPyB.parseString(
strScreeningObtained)
strXMLScreeningObtained = xsDataScreeningObtained.marshal()
EDAssert.equal(strXMLScreeningExpected, strXMLScreeningObtained)
#Compare screeningInput
strInputScreeningInputExpected = self.readAndParseFile(
self.__strReferenceInputScreeningInput)
xsDataScreeningInputExpected = XSDataInputISPyB.parseString(
strInputScreeningInputExpected)
strXMLScreeningInputExpected = xsDataScreeningInputExpected.marshal()
strScreeningInputObtained = self.readAndParseFile(
self.__strObtainedScreeningInput)
xsDataScreeningInputObtained = XSDataInputISPyB.parseString(
strScreeningInputObtained)
strXMLScreeningInputObtained = xsDataScreeningInputObtained.marshal()
EDAssert.equal(strXMLScreeningInputExpected,
strXMLScreeningInputObtained)
#Compare screeningOutputContainer
strInputScreeningOutputContainerExpected = self.readAndParseFile(
self.__strReferenceInputScreeningOutputContainer)
xsDataScreeningOutputContainerExpected = XSDataInputISPyB.parseString(
strInputScreeningOutputContainerExpected)
strXMLScreeningOutputContainerExpected = xsDataScreeningOutputContainerExpected.marshal(
)
strScreeningOutputContainerObtained = self.readAndParseFile(
self.__strObtainedScreeningOutputContainer)
xsDataScreeningOutputContainerObtained = XSDataInputISPyB.parseString(
strScreeningOutputContainerObtained)
strXMLScreeningOutputContainerObtained = xsDataScreeningOutputContainerObtained.marshal(
)
EDAssert.equal(strXMLScreeningOutputContainerExpected,
strXMLScreeningOutputContainerObtained)
#Compare screeningRank
strInputScreeningRankExpected = self.readAndParseFile(
self.__strReferenceInputScreeningRank)
xsDataScreeningRankExpected = XSDataInputISPyB.parseString(
strInputScreeningRankExpected)
strXMLScreeningRankExpected = xsDataScreeningRankExpected.marshal()
strScreeningRankObtained = self.readAndParseFile(
self.__strObtainedScreeningRank)
xsDataScreeningRankObtained = XSDataInputISPyB.parseString(
strScreeningRankObtained)
strXMLScreeningRankObtained = xsDataScreeningRankObtained.marshal()
EDAssert.equal(strXMLScreeningRankExpected,
strXMLScreeningRankObtained)
#Compare screeningRankSet
strInputScreeningRankSetExpected = self.readAndParseFile(
self.__strReferenceInputScreeningRankSet)
xsDataScreeningRankSetExpected = XSDataInputISPyB.parseString(
strInputScreeningRankSetExpected)
strXMLScreeningRankSetExpected = xsDataScreeningRankSetExpected.marshal(
)
strScreeningRankSetObtained = self.readAndParseFile(
self.__strObtainedScreeningRankSet)
xsDataScreeningRankSetObtained = XSDataInputISPyB.parseString(
strScreeningRankSetObtained)
strXMLScreeningRankSetObtained = xsDataScreeningRankSetObtained.marshal(
)
EDAssert.equal(strXMLScreeningRankSetExpected,
strXMLScreeningRankSetObtained)
#Compare screeningFile
strInputScreeningFileExpected = self.readAndParseFile(
self.__strReferenceInputScreeningFile)
xsDataScreeningFileExpected = XSDataInputISPyB.parseString(
strInputScreeningFileExpected)
strXMLScreeningFileExpected = xsDataScreeningFileExpected.marshal()
strScreeningFileObtained = self.readAndParseFile(
self.__strObtainedScreeningFile)
xsDataScreeningFileObtained = XSDataInputISPyB.parseString(
strScreeningFileObtained)
strXMLScreeningFileObtained = xsDataScreeningFileObtained.marshal()
EDAssert.equal(strXMLScreeningFileExpected,
strXMLScreeningFileObtained)
self.cleanUp(edPluginISPyB)
def parseLabelitScreenOutput(self, _strLabelitLogText):
"""
This method parses the labelit.screen log and populates the relevant
parts of the XSDataLabelitScreenOutput object which is then returned.
"""
self.DEBUG("EDPluginLabelitIndexingv1_1.parseLabelitLogText")
xsDataLabelitScreenOutput = None
iIndex = 0
listLogLines = _strLabelitLogText.split('\n')
bFoundLabelitIndexingResults = False
bContinue = True
while (bContinue):
if (listLogLines[iIndex].find("LABELIT Indexing results:") != -1):
bFoundLabelitIndexingResults = True
bContinue = False
else:
if (iIndex < (len(listLogLines) - 2)):
iIndex += 1
else:
bContinue = False
if (bFoundLabelitIndexingResults == False):
strErrorMessage = "EDPluginLabelitIndexingv1_1.parseLabelitLogText : Labelit log message: %s" % _strLabelitLogText
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
else:
# We found some indexing results!
xsDataLabelitScreenOutput = XSDataLabelitScreenOutput()
iIndex += 1
strBeamDistanceMosaicity = listLogLines[iIndex]
strBeamDistanceMosaicity = strBeamDistanceMosaicity.replace(
',', '')
strBeamDistanceMosaicity = strBeamDistanceMosaicity.replace(
'mm', ' ')
strBeamDistanceMosaicity = strBeamDistanceMosaicity.replace(
'=', ' ')
listStringBeamDistanceMosaicity = strBeamDistanceMosaicity.split()
fBeamX = float(listStringBeamDistanceMosaicity[3])
fBeamY = float(listStringBeamDistanceMosaicity[5])
xsDataLabelitScreenOutput.setBeamCentreX(XSDataLength(fBeamX))
xsDataLabelitScreenOutput.setBeamCentreY(XSDataLength(fBeamY))
fDistance = float(listStringBeamDistanceMosaicity[7])
xsDataLabelitScreenOutput.setDistance(XSDataLength(fDistance))
fMosaicity = float(listStringBeamDistanceMosaicity[11])
xsDataLabelitScreenOutput.setMosaicity(XSDataAngle(fMosaicity))
iIndex += 3
bContinue = True
bFoundSelectedSolution = False
while (bContinue):
xsDataLabelitScreenSolution = XSDataLabelitScreenSolution()
strLabelitSolution = listLogLines[iIndex]
listStringLabelitSolution = strLabelitSolution.split()
if (listStringLabelitSolution[0] == ":)"):
xsDataLabelitScreenSolution.setHappy(XSDataBoolean(True))
elif (listStringLabelitSolution[0] == ":("):
xsDataLabelitScreenSolution.setHappy(XSDataBoolean(False))
elif (listStringLabelitSolution[0] == ";("):
xsDataLabelitScreenSolution.setHappy(XSDataBoolean(False))
else:
# We have an error...
strErrorMessage = "Indexing with labelit.screen failed! Log file:"
self.ERROR(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.ERROR(_strLabelitLogText)
self.addErrorMessage(_strLabelitLogText)
self.setFailure()
bContinue = False
if bContinue:
iSolutionNumber = int(listStringLabelitSolution[1])
xsDataLabelitScreenSolution.setSolutionNumber(
XSDataInteger(iSolutionNumber))
fMetricFitValue = float(listStringLabelitSolution[2])
xsDataLabelitScreenSolution.setMetricFitValue(
XSDataDouble(fMetricFitValue))
strMetricFitCode = listStringLabelitSolution[3]
xsDataLabelitScreenSolution.setMetricFitCode(
XSDataString(strMetricFitCode))
fRMSD = float(listStringLabelitSolution[4])
xsDataLabelitScreenSolution.setRmsd(XSDataLength(fRMSD))
iNumberOfSpots = int(listStringLabelitSolution[5])
xsDataLabelitScreenSolution.setNumberOfSpots(
XSDataInteger(iNumberOfSpots))
strCrystalSystem = listStringLabelitSolution[6]
strBravaisLattice = listStringLabelitSolution[7]
xsDataLabelitScreenSolution.setCrystalSystem(
XSDataString(strCrystalSystem))
xsDataLabelitScreenSolution.setBravaisLattice(
XSDataString(strBravaisLattice))
xsDataCell = XSDataCell()
xsDataCell.setLength_a(
XSDataLength(float(listStringLabelitSolution[8])))
xsDataCell.setLength_b(
XSDataLength(float(listStringLabelitSolution[9])))
xsDataCell.setLength_c(
XSDataLength(float(listStringLabelitSolution[10])))
xsDataCell.setAngle_alpha(
XSDataAngle(float(listStringLabelitSolution[11])))
xsDataCell.setAngle_beta(
XSDataAngle(float(listStringLabelitSolution[12])))
xsDataCell.setAngle_gamma(
XSDataAngle(float(listStringLabelitSolution[13])))
xsDataLabelitScreenSolution.setUnitCell(xsDataCell)
iVolume = int(listStringLabelitSolution[14])
xsDataLabelitScreenSolution.setVolume(
XSDataInteger(iVolume))
# Find the selected solution
if (bFoundSelectedSolution == False):
if (xsDataLabelitScreenSolution.getHappy().getValue()
== True):
# Check if forced space group
if self.__strForcedSpaceGroup == None or self.__strForcedSpaceGroup == "":
bFoundSelectedSolution = True
xsDataLabelitScreenOutput.setSelectedSolutionNumber(
XSDataInteger(iSolutionNumber))
else:
listBravaisLattice = self.getBravaisLatticeFromSpaceGroup(
self.__strForcedSpaceGroup)
for strPossibleBravaisLattice in listBravaisLattice:
if strBravaisLattice == strPossibleBravaisLattice:
bFoundSelectedSolution = True
xsDataLabelitScreenOutput.setSelectedSolutionNumber(
XSDataInteger(iSolutionNumber))
xsDataLabelitScreenOutput.addLabelitScreenSolution(
xsDataLabelitScreenSolution)
if (iIndex < (len(listLogLines) - 2)):
iIndex += 1
else:
bContinue = False
return xsDataLabelitScreenOutput
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlAlignStackv1_0.process")
for iFrame in self.iFrames:
edPluginExecAccumulator = self.loadPlugin(
self.__strControlledPluginAccumulator)
queryRaw = XSDataQuery()
queryShift = XSDataQuery()
queryRaw.setRemoveItems(XSDataBoolean(False))
queryShift.setRemoveItems(XSDataBoolean(False))
xsdataAcc = XSDataInputAccumulator()
if (EDPluginControlAlignStackv1_0.__iRefFrame
== iFrame) or (self.bDoAlign == False):
EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame] = (0.0,
0.0)
EDPluginControlAlignStackv1_0.__dictRelShift[iFrame] = (0.0,
0.0)
self.hdf5_offset(index=iFrame, offset=[0.0, 0.0])
edPluginExecShift = self.loadPlugin(
self.__strControlledPluginShift)
xsdata = XSDataInputShiftImage(
index=XSDataInteger(iFrame),
offset=[
XSDataDouble(i) for i in
EDPluginControlAlignStackv1_0.__dictAbsShift[iFrame]
],
inputImage=self.getFrameRef(iFrame),
outputImage=XSDataImageExt(
shared=XSDataString("Shifted-%06i" % iFrame)))
edPluginExecShift.setDataInput(xsdata)
edPluginExecShift.connectSUCCESS(self.doSuccessExecShiftImage)
edPluginExecShift.connectFAILURE(self.doFailureExecShiftImage)
self.queue.put(edPluginExecShift)
if (self.bDoAlign == False):
self.executeControlledPlugins()
return
elif EDPluginControlAlignStackv1_0.__iRefFrame < iFrame:
if self.bAlwaysMOvsRef:
queryRaw.setItem([
XSDataString(
"raw %04i" %
(EDPluginControlAlignStackv1_0.__iRefFrame)),
XSDataString("raw %04i" % iFrame)
])
xsdataAcc.setQuery([queryRaw])
else:
queryRaw.setItem([
XSDataString("raw %04i" % (iFrame - 1)),
XSDataString("raw %04i" % iFrame)
])
queryShift.setItem([
XSDataString("shift %04i" % i) for i in range(
EDPluginControlAlignStackv1_0.__iRefFrame +
1, iFrame + 1)
])
xsdataAcc.setQuery([queryRaw, queryShift])
elif EDPluginControlAlignStackv1_0.__iRefFrame > iFrame:
if self.bAlwaysMOvsRef:
queryRaw.setItem([
XSDataString("raw %04i" % iFrame),
XSDataString(
"raw %04i" %
(EDPluginControlAlignStackv1_0.__iRefFrame))
])
xsdataAcc.setQuery([queryRaw])
else:
queryRaw.setItem([
XSDataString("raw %04i" % (iFrame + 1)),
XSDataString("raw %04i" % iFrame)
])
queryShift.setItem([
XSDataString("shift %04i" % i) for i in range(
EDPluginControlAlignStackv1_0.__iRefFrame -
1, iFrame - 1, -1)
])
xsdataAcc.setQuery([queryRaw, queryShift])
if (EDPluginControlAlignStackv1_0.__iRefFrame == iFrame):
self.saveReferenceFrame(iFrame)
xsdataAcc.setItem([XSDataString("raw %04i" % iFrame)])
edPluginExecAccumulator.setDataInput(xsdataAcc)
edPluginExecAccumulator.connectSUCCESS(
self.doSuccessExecAccumultor)
edPluginExecAccumulator.connectFAILURE(
self.doFailureExecAccumulator)
self.queue.put(edPluginExecAccumulator)
self.executeControlledPlugins()
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalCondition':
obj_ = XSDataExperimentalCondition()
obj_.build(child_)
self.setExperimentalCondition(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'diffractionPlan':
obj_ = XSDataDiffractionPlan()
obj_.build(child_)
self.setDiffractionPlan(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'sample':
obj_ = XSDataSampleCrystalMM()
obj_.build(child_)
self.setSample(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'imagePath':
obj_ = XSDataFile()
obj_.build(child_)
self.imagePath.append(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'flux':
obj_ = XSDataFloat()
obj_.build(child_)
self.setFlux(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'minExposureTimePerImage':
obj_ = XSDataTime()
obj_.build(child_)
self.setMinExposureTimePerImage(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSize':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSize(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSizeX':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSizeX(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSizeY':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSizeY(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'templateMode':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setTemplateMode(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'generatedTemplateFile':
obj_ = XSDataFile()
obj_.build(child_)
self.setGeneratedTemplateFile(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'resultsFilePath':
obj_ = XSDataFile()
obj_.build(child_)
self.setResultsFilePath(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamPosX':
obj_ = XSDataFloat()
obj_.build(child_)
self.setBeamPosX(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamPosY':
obj_ = XSDataFloat()
obj_.build(child_)
self.setBeamPosY(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'wavelength':
obj_ = XSDataWavelength()
obj_.build(child_)
self.setWavelength(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'transmission':
obj_ = XSDataDouble()
obj_.build(child_)
self.setTransmission(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'dataCollectionId':
obj_ = XSDataInteger()
obj_.build(child_)
self.setDataCollectionId(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'shortComments':
obj_ = XSDataString()
obj_.build(child_)
self.setShortComments(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'comments':
obj_ = XSDataString()
obj_.build(child_)
self.setComments(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'inputCharacterisation':
obj_ = XSDataInputCharacterisation()
obj_.build(child_)
self.setInputCharacterisation(obj_)
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'dataCollectionId':
obj_ = XSDataInteger()
obj_.build(child_)
self.setDataCollectionId(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'dirN':
obj_ = XSDataFile()
obj_.build(child_)
self.setDirN(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'configDef':
obj_ = XSDataFile()
obj_.build(child_)
self.setConfigDef(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'templateN':
obj_ = XSDataString()
obj_.build(child_)
self.setTemplateN(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'fromN':
obj_ = XSDataInteger()
obj_.build(child_)
self.setFromN(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'toN':
obj_ = XSDataInteger()
obj_.build(child_)
self.setToN(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'processDirectory':
obj_ = XSDataFile()
obj_.build(child_)
self.setProcessDirectory(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'doAnom':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setDoAnom(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'doAnomAndNonanom':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setDoAnomAndNonanom(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'symm':
obj_ = XSDataString()
obj_.build(child_)
self.setSymm(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'cell':
obj_ = XSDataString()
obj_.build(child_)
self.setCell(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'reprocess':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setReprocess(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'lowResolutionLimit':
obj_ = XSDataDouble()
obj_.build(child_)
self.setLowResolutionLimit(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'highResolutionLimit':
obj_ = XSDataDouble()
obj_.build(child_)
self.setHighResolutionLimit(obj_)
XSDataInput.buildChildren(self, child_, nodeName_)
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'template':
obj_ = XSDataString()
obj_.build(child_)
self.setTemplate(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'first_image_number':
obj_ = XSDataInteger()
obj_.build(child_)
self.setFirst_image_number(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'last_image_number':
obj_ = XSDataInteger()
obj_.build(child_)
self.setLast_image_number(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'first_run_number':
obj_ = XSDataInteger()
obj_.build(child_)
self.setFirst_run_number(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'last_run_number':
obj_ = XSDataInteger()
obj_.build(child_)
self.setLast_run_number(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'line_number_of':
obj_ = XSDataInteger()
obj_.build(child_)
self.setLine_number_of(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'reversing_rotation':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setReversing_rotation(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'pixelMin':
obj_ = XSDataInteger()
obj_.build(child_)
self.setPixelMin(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'pixelMax':
obj_ = XSDataInteger()
obj_.build(child_)
self.setPixelMax(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamstopDirection':
obj_ = XSDataString()
obj_.build(child_)
self.setBeamstopDirection(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamstopSize':
obj_ = XSDataDouble()
obj_.build(child_)
self.setBeamstopSize(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamstopDistance':
obj_ = XSDataDouble()
obj_.build(child_)
self.setBeamstopDistance(obj_)
XSDataInput.buildChildren(self, child_, nodeName_)
strPathToTempDir = tempfile.mkdtemp(
prefix="edna-image-quality-indicators_")
os.chdir(strPathToTempDir)
EDVerbose.setLogFileName(os.path.join(strPathToTempDir, "edna.log"))
# Popolate input data
EDVerbose.screen("Starting EDNA image quality indicators processing")
EDVerbose.screen("Arguments: %r" % sys.argv)
bVerbose = False
bDebug = False
listPaths = []
for iIndex, strArg in enumerate(sys.argv[1:]):
strarg = strArg.lower()
if strarg == "--verbose":
EDVerbose.setVerboseOn()
elif strarg == "--debug":
EDVerbose.setVerboseDebugOn()
if os.path.exists(strArg):
listPaths.append(os.path.abspath(strArg))
for strPath in listPaths:
xsDataInputGridScreening = XSDataInputGridScreening()
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(strPath))
xsDataInputGridScreening.setImageFile(xsDataFile)
xsDataInputGridScreening.setDoOnlyImageQualityIndicators(
XSDataBoolean(True))
xsDataInputGridScreening.setStoreImageQualityIndicatorsInISPyB(
XSDataBoolean(True))
edPluginGridScreening = EDFactoryPluginStatic.loadPlugin(
"EDPluginControlGridScreeningv1_0")
edPluginGridScreening.setDataInput(xsDataInputGridScreening)
edPluginGridScreening.executeSynchronous()
def createInputCharacterisationFromSubWedges(self):
EDVerbose.DEBUG(
"EDPluginControlInterfacev1_2.createInputCharacterisationFromSubWedges"
)
xsDataResultSubWedgeAssemble = self.edPluginControlSubWedgeAssemble.getDataOutput(
)
self.xsDataInputCharacterisation = XSDataInputCharacterisation()
xsDataCollection = XSDataCollection()
# Default exposure time (for the moment, this value should be
# possible to read from the command line)
if self.xsDataDiffractionPlan is None:
self.xsDataDiffractionPlan = XSDataDiffractionPlan()
if (not xsDataResultSubWedgeAssemble is None):
pyListSubWedge = xsDataResultSubWedgeAssemble.getSubWedge()
xsDataCollection.setSubWedge(pyListSubWedge)
for xsDataSubWedge in pyListSubWedge:
if (self.strComplexity is not None):
self.xsDataDiffractionPlan.setComplexity(
XSDataString(self.strComplexity))
if (self.fFlux is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setFlux(XSDataFlux(self.fFlux))
if (self.fBeamSizeX is not None) and (self.fBeamSizeY
is not None):
xsDataSize = XSDataSize()
xsDataSize.setX(XSDataLength(self.fBeamSizeX))
xsDataSize.setY(XSDataLength(self.fBeamSizeY))
xsDataSubWedge.getExperimentalCondition().getBeam(
).setSize(xsDataSize)
if (self.fBeamPosX is not None):
xsDataSubWedge.getExperimentalCondition().getDetector(
).setBeamPositionX(XSDataLength(self.fBeamPosX))
if (self.fBeamPosY is not None):
xsDataSubWedge.getExperimentalCondition().getDetector(
).setBeamPositionY(XSDataLength(self.fBeamPosY))
if (self.fMinExposureTimePerImage is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setMinExposureTimePerImage(
XSDataTime(self.fMinExposureTimePerImage))
if (self.fTransmission is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setTransmission(XSDataDouble(self.fTransmission))
if (self.fWavelength is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setWavelength(XSDataWavelength(self.fWavelength))
if self.fMinOscillationWidth != None:
xsDataSubWedge.getExperimentalCondition().getGoniostat(
).setMinOscillationWidth(
XSDataAngle(self.fMinOscillationWidth))
if self.fMaxOscillationSpeed != None:
xsDataSubWedge.getExperimentalCondition().getGoniostat(
).setMaxOscillationSpeed(
XSDataAngularSpeed(self.fMaxOscillationSpeed))
if (self.strForcedSpaceGroup is not None):
self.xsDataDiffractionPlan.setForcedSpaceGroup(
XSDataString(self.strForcedSpaceGroup))
self.xsDataDiffractionPlan.setAnomalousData(
XSDataBoolean(self.bAnomalousData))
self.xsDataDiffractionPlan.setMaxExposureTimePerDataCollection(
XSDataTime(self.fMaxExposureTimePerDataCollection))
if (self.strStrategyOption is not None):
self.xsDataDiffractionPlan.setStrategyOption(
XSDataString(self.strStrategyOption))
xsDataCollection.setDiffractionPlan(self.xsDataDiffractionPlan)
if self.xsDataSample is not None:
xsDataCollection.setSample(
XSDataSampleCrystalMM.parseString(self.xsDataSample.marshal()))
self.xsDataInputCharacterisation.setDataCollection(xsDataCollection)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
if self.dataInput.startImageNumber is not None:
imageNoStart = self.dataInput.startImageNumber.value
if self.dataInput.endImageNumber is not None:
imageNoEnd = self.dataInput.endImageNumber.value
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if imageNoStart is None:
imageNoStart = ispybDataCollection.startImageNumber
if imageNoEnd is None:
imageNoEnd = ispybDataCollection.startImageNumber + \
ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory,
fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XDSAPP", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(
self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
if self.dataInput.hdf5ToCbfDirectory is not None:
dir = self.dataInput.hdf5ToCbfDirectory.path.value
pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]
self.timeStart = time.localtime()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
xsDataInputXDSAPPAnom.image = XSDataFile(
XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPAnom.spacegroup = XSDataString(
self.xdsAppSpacegroup)
self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
self.edPluginExecXDSAPPAnom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
xsDataInputXDSAPPNoanom.image = XSDataFile(
XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPNoanom.spacegroup = XSDataString(
self.xdsAppSpacegroup)
self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
self.edPluginExecXDSAPPNoanom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXDSAPPAnom.synchronize()
xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
# Run XSCALE even if XSCALE.LP is present
strPathXscaleLpAnom = self.runXscale(
self.edPluginExecXDSAPPAnom.getWorkingDirectory(),
anom=True,
merged=True)
if self.doNoanom:
self.edPluginExecXDSAPPNoanom.synchronize()
xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
strPathXscaleLpNoanom = self.runXscale(
self.edPluginExecXDSAPPNoanom.getWorkingDirectory(),
anom=False,
merged=True)
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.dataInput.dataCollectionId is not None:
# Check if we should use XDS_ASCII_to_XML.pl
if self.doAnom:
if self.useXdsAsciiToXml:
# Only for anom runs
self.runXdsAsciiToXml(
xsDataResultXDSAPPAnom,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom,
self.autoProcProgramIdAnom)
else:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
xsDataResultXDSAPPAnom, processDirectory, template,
strPathXscaleLpAnom, True, proposal, self.timeStart,
self.timeEnd, self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom,
self.autoProcProgramIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
xsDataResultXDSAPPNoanom, processDirectory, template,
strPathXscaleLpNoanom, False, proposal, self.timeStart,
self.timeEnd, self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdNoanom,
self.autoProcProgramIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'title':
obj_ = XSDataString()
obj_.build(child_)
self.setTitle(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataQ':
obj_ = XSDataDouble()
obj_.build(child_)
self.experimentalDataQ.append(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataQArray':
obj_ = XSDataArray()
obj_.build(child_)
self.setExperimentalDataQArray(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataValues':
obj_ = XSDataDouble()
obj_.build(child_)
self.experimentalDataValues.append(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataIArray':
obj_ = XSDataArray()
obj_.build(child_)
self.setExperimentalDataIArray(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataStdDev':
obj_ = XSDataDouble()
obj_.build(child_)
self.experimentalDataStdDev.append(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataStdArray':
obj_ = XSDataArray()
obj_.build(child_)
self.setExperimentalDataStdArray(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalDataFile':
obj_ = XSDataFile()
obj_.build(child_)
self.setExperimentalDataFile(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'rMaxSearchSettings':
obj_ = XSDataSolutionScatteringSettings()
obj_.build(child_)
self.setRMaxSearchSettings(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'angularUnits':
obj_ = XSDataInteger()
obj_.build(child_)
self.setAngularUnits(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'symmetry':
obj_ = XSDataString()
obj_.build(child_)
self.setSymmetry(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'mode':
obj_ = XSDataString()
obj_.build(child_)
self.setMode(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'iNbThreads':
obj_ = XSDataInteger()
obj_.build(child_)
self.setINbThreads(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'onlyGnom':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setOnlyGnom(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'plotFit':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setPlotFit(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'qMin':
obj_ = XSDataDouble()
obj_.build(child_)
self.setQMin(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'qMax':
obj_ = XSDataDouble()
obj_.build(child_)
self.setQMax(obj_)
XSDataInput.buildChildren(self, child_, nodeName_)
def generateLegacyXSDataISPyBStrategy(
_xsDataResultCharacterisation,
_xsDataISPyBScreeningOutputContainer):
# Anomalus data
bAnomalousData = False
xsDataCollection = _xsDataResultCharacterisation.dataCollection
if xsDataCollection is not None:
xsDataDiffractionPlan = xsDataCollection.diffractionPlan
if xsDataDiffractionPlan is not None:
if xsDataDiffractionPlan.anomalousData is not None:
bAnomalousData = xsDataDiffractionPlan.getAnomalousData(
).getValue()
# Use the existing xsDataISPyBScreeningStrategy in the _xsDataISPyBScreeningOutputContainer for the first sub wedge:
if len(_xsDataISPyBScreeningOutputContainer.screeningStrategyContainer
) > 0:
xsDataISPyBScreeningStrategyContainerFirst = _xsDataISPyBScreeningOutputContainer.screeningStrategyContainer[
0]
xsDataISPyBScreeningStrategyFirst = xsDataISPyBScreeningStrategyContainerFirst.screeningStrategy
# Strategy information
bFirstStrategy = True
xsDataResultStrategy = _xsDataResultCharacterisation.getStrategyResult(
)
if (xsDataResultStrategy is not None):
pyListXSDataCollectionPlan = xsDataResultStrategy.getCollectionPlan(
)
if (pyListXSDataCollectionPlan is not None):
for xsDataCollectionPlan in pyListXSDataCollectionPlan:
iCollectionPlanNumber = xsDataCollectionPlan.getCollectionPlanNumber(
).getValue()
pyStrCollectionPlanComment = None
if (xsDataCollectionPlan.getComment() is not None):
pyStrCollectionPlanComment = xsDataCollectionPlan.getComment(
).getValue()
fCompleteness = None
fMultiplicity = None
fResolution = None
fRankingResolution = None
fTransmission = None
xsDataStrategySummary = xsDataCollectionPlan.getStrategySummary(
)
if (xsDataStrategySummary is not None):
if (xsDataStrategySummary.getCompleteness()
is not None):
fCompleteness = xsDataStrategySummary.getCompleteness(
).getValue()
if (xsDataStrategySummary.getRedundancy()
is not None):
fMultiplicity = xsDataStrategySummary.getRedundancy(
).getValue()
if (xsDataStrategySummary.getResolution()
is not None):
fResolution = xsDataStrategySummary.getResolution(
).getValue()
if (xsDataStrategySummary.getRankingResolution()
is not None):
fRankingResolution = xsDataStrategySummary.getRankingResolution(
).getValue()
xsDataCollectionStrategy = xsDataCollectionPlan.getCollectionStrategy(
)
if (xsDataCollectionStrategy is not None):
pyListXSDataSubWedge = xsDataCollectionStrategy.getSubWedge(
)
if (pyListXSDataSubWedge is not None):
for xsDataSubWedge in pyListXSDataSubWedge:
iSubWedgeNumber = xsDataSubWedge.getSubWedgeNumber(
).getValue()
if bFirstStrategy:
xsDataISPyBScreeningStrategy = xsDataISPyBScreeningStrategyFirst
else:
xsDataISPyBScreeningStrategy = XSDataISPyBScreeningStrategy(
)
xsDataISPyBScreeningStrategyContainer = XSDataISPyBScreeningStrategyContainer(
)
xsDataISPyBScreeningStrategyContainer.screeningStrategy = xsDataISPyBScreeningStrategy
fPhiStart = xsDataSubWedge.getExperimentalCondition(
).getGoniostat().getRotationAxisStart(
).getValue()
fPhiEnd = xsDataSubWedge.getExperimentalCondition(
).getGoniostat().getRotationAxisEnd(
).getValue()
fRotation = xsDataSubWedge.getExperimentalCondition(
).getGoniostat().getOscillationWidth(
).getValue()
fExposureTime = xsDataSubWedge.getExperimentalCondition(
).getBeam().getExposureTime().getValue()
fTransmission = None
if xsDataSubWedge.experimentalCondition.beam.transmission:
fTransmission = xsDataSubWedge.experimentalCondition.beam.transmission.value
pyStrProgram = "BEST: Wedge no %d," % iCollectionPlanNumber
if (pyStrCollectionPlanComment
is not None):
pyStrProgram += " ( %s )" % pyStrCollectionPlanComment
pyStrProgram += " sub wedge no %d" % iSubWedgeNumber
xsDataISPyBScreeningStrategy.setPhiStart(
XSDataDouble(fPhiStart))
xsDataISPyBScreeningStrategy.setPhiEnd(
XSDataDouble(fPhiEnd))
xsDataISPyBScreeningStrategy.setRotation(
XSDataDouble(fRotation))
xsDataISPyBScreeningStrategy.setExposureTime(
XSDataDouble(fExposureTime))
if fTransmission is not None:
xsDataISPyBScreeningStrategy.transmission = XSDataDouble(
fTransmission)
if (fCompleteness is not None):
xsDataISPyBScreeningStrategy.setCompleteness(
XSDataDouble(fCompleteness))
if (fMultiplicity is not None):
xsDataISPyBScreeningStrategy.setMultiplicity(
XSDataDouble(fMultiplicity))
if (fResolution is not None):
xsDataISPyBScreeningStrategy.setResolution(
XSDataDouble(fResolution))
if (fRankingResolution is not None):
xsDataISPyBScreeningStrategy.setRankingResolution(
XSDataDouble(fRankingResolution))
if (bAnomalousData is not None):
xsDataISPyBScreeningStrategy.setAnomalous(
XSDataBoolean(bAnomalousData))
else:
xsDataISPyBScreeningStrategy.setAnomalous(
XSDataBoolean(False))
xsDataISPyBScreeningStrategy.setProgram(
XSDataString(pyStrProgram))
if bFirstStrategy:
bFirstStrategy = False
else:
_xsDataISPyBScreeningOutputContainer.addScreeningStrategyContainer(
xsDataISPyBScreeningStrategyContainer
)
def setPluginInput(self, _edPlugin):
xsDataDiffractionPlan = XSDataDiffractionPlan()
if (not self.__fMaxExposureTimePerDataCollection is None):
xsDataDiffractionPlan.setMaxExposureTimePerDataCollection(
XSDataTime(self.__fMaxExposureTimePerDataCollection))
if (not self.__strForcedSpaceGroup is None):
xsDataDiffractionPlan.setForcedSpaceGroup(
XSDataString(self.__strForcedSpaceGroup))
if (not self.__bAnomalousData is None):
xsDataDiffractionPlan.setAnomalousData(
XSDataBoolean(self.__bAnomalousData))
if (not self.__strStrategyOption is None):
xsDataDiffractionPlan.setStrategyOption(
XSDataString(self.__strStrategyOption))
if (not self.__fDoseLimit is None):
xsDataDiffractionPlan.setDoseLimit(XSDataDouble(self.__fDoseLimit))
if (not self.__fAimedCompleteness is None):
xsDataDiffractionPlan.setAimedCompleteness(
XSDataDouble(self.__fAimedCompleteness))
if (not self.__fAimedMultiplicity is None):
xsDataDiffractionPlan.setAimedMultiplicity(
XSDataDouble(self.__fAimedMultiplicity))
if (not self.__fAimedResolution is None):
xsDataDiffractionPlan.setAimedResolution(
XSDataDouble(self.__fAimedResolution))
if (not self.__strComplexity is None):
xsDataDiffractionPlan.setComplexity(
XSDataString(self.__strComplexity))
_edPlugin.setDataInput(xsDataDiffractionPlan, "diffractionPlan")
if (not self.__listImagePaths is None):
for strImagePath in self.__listImagePaths:
_edPlugin.setDataInput(XSDataString(strImagePath),
"imagePaths")
if (not self.__xsDataInputCharacterisation is None):
_edPlugin.setDataInput(self.__xsDataInputCharacterisation,
"inputCharacterisation")
if (not self.__fFlux is None):
_edPlugin.setDataInput(XSDataFloat(self.__fFlux), "flux")
if (not self.__fMinExposureTimePerImage is None):
_edPlugin.setDataInput(
XSDataFloat(self.__fMinExposureTimePerImage),
"minExposureTimePerImage")
if (not self.__fBeamSize is None):
_edPlugin.setDataInput(XSDataFloat(self.__fBeamSize), "beamSize")
if (not self.__bTemplateMode is None):
_edPlugin.setDataInput(XSDataBoolean(self.__bTemplateMode),
"templateMode")
if (not self.__strGeneratedTemplateFile is None):
_edPlugin.setDataInput(
XSDataString(self.__strGeneratedTemplateFile),
"generatedTemplateFile")
if (not self.__strResultsFilePath is None):
_edPlugin.setDataInput(XSDataString(self.__strResultsFilePath),
"resultsFilePath")
if (not self.__fBeamPosX is None):
_edPlugin.setDataInput(XSDataFloat(self.__fBeamPosX), "beamPosX")
if (not self.__fBeamPosY is None):
_edPlugin.setDataInput(XSDataFloat(self.__fBeamPosY), "beamPosY")
if (not self.__iDataCollectionId is None):
_edPlugin.setDataInput(XSDataInteger(self.__iDataCollectionId),
"dataCollectionId")
if (not self.__strShortComments is None):
_edPlugin.setDataInput(XSDataString(self.__strShortComments),
"shortComments")
if (not self.__strComments is None):
_edPlugin.setDataInput(XSDataString(self.__strComments),
"comments")
if (not self.__fTransmission is None):
_edPlugin.setDataInput(XSDataDouble(self.__fTransmission),
"transmission")
def process(self, _edPlugin=None):
EDPluginExec.process(self)
# Wait for file if it's not already on disk'
finalSize = None
hasTimedOut = False
shouldContinue = True
fileDir = os.path.dirname(self.strFilePath)
if os.path.exists(fileDir):
# Patch provided by Sebastien 2018/02/09 for forcing NFS cache:
self.DEBUG(
"NFS cache clear, doing os.fstat on directory {0}".format(
fileDir))
d = os.open(fileDir, os.O_DIRECTORY)
statResult = os.fstat(d)
self.DEBUG("Results of os.fstat: {0}".format(statResult))
# Check if file is there
if os.path.exists(self.strFilePath):
fileSize = os.path.getsize(self.strFilePath)
if self.expectedSize is not None:
# Check size
if fileSize > self.expectedSize:
shouldContinue = False
finalSize = fileSize
if shouldContinue:
self.screen("Waiting %d seconds for file %s" %
(self.timeOut, self.strFilePath))
#
timeStart = time.time()
while shouldContinue and not hasTimedOut:
# Sleep 1 s
time.sleep(1)
if os.path.exists(fileDir):
# Patch provided by Sebastien 2018/02/09 for forcing NFS cache:
self.DEBUG(
"NFS cache clear, doing os.fstat on directory {0}".
format(fileDir))
d = os.open(fileDir, os.O_DIRECTORY)
statResult = os.fstat(d)
self.DEBUG("Results of os.fstat: {0}".format(statResult))
# Try to execute a "ls" on the file directory - this sometimes speed up NFS
#if os.path.exists(fileDir):
# os.system("ls %s > /dev/null" % fileDir)
timeElapsed = time.time() - timeStart
# Check if time out
if timeElapsed > self.timeOut:
hasTimedOut = True
strWarning = "Timeout while waiting for file %s" % self.strFilePath
self.WARNING(strWarning)
self.addWarningMessage(strWarning)
else:
# Check if file is there
if os.path.exists(self.strFilePath):
fileSize = os.path.getsize(self.strFilePath)
if self.expectedSize is not None:
# Check that it has right size
if fileSize > self.expectedSize:
shouldContinue = False
else:
shouldContinue = False
finalSize = fileSize
self.dataOutput.timedOut = XSDataBoolean(hasTimedOut)
if finalSize is not None:
self.dataOutput.finalSize = XSDataInteger(finalSize)
def generateXSDataISPyBScreeningOutputContainer(
_xsDataResultCharacterisation, _strStatusMessage, _fKappa, _fPhi):
xsDataISPyBScreeningOutput = XSDataISPyBScreeningOutput()
xsDataISPyBScreeningOutput.program = XSDataString("EDNA MXv1")
xsDataISPyBScreeningOutputLattice = None
# Indexing information
bSuccessfulIndexing = False
xsDataIndexingResult = _xsDataResultCharacterisation.getIndexingResult(
)
if (xsDataIndexingResult is not None):
xsDataIndexingSolutionSelected = xsDataIndexingResult.getSelectedSolution(
)
if (xsDataIndexingSolutionSelected is not None):
xsDataISPyBScreeningOutputLattice = XSDataISPyBScreeningOutputLattice(
)
bSuccessfulIndexing = True
xsDataStatisticsIndexing = xsDataIndexingSolutionSelected.getStatistics(
)
if (xsDataStatisticsIndexing is not None):
xsDataISPyBScreeningOutput.beamShiftX = xsDataStatisticsIndexing.beamPositionShiftX
xsDataISPyBScreeningOutput.beamShiftY = xsDataStatisticsIndexing.beamPositionShiftY
xsDataISPyBScreeningOutput.spotDeviationR = xsDataStatisticsIndexing.spotDeviationPositional
xsDataISPyBScreeningOutput.spotDeviationTheta = xsDataStatisticsIndexing.spotDeviationAngular
if ((xsDataStatisticsIndexing.spotsTotal is not None) and
(xsDataStatisticsIndexing.spotsUsed is not None)):
iSpotsTotal = xsDataStatisticsIndexing.spotsTotal.value
iSpotsUsed = xsDataStatisticsIndexing.spotsUsed.value
xsDataISPyBScreeningOutput.numSpotsFound = xsDataStatisticsIndexing.spotsTotal
xsDataISPyBScreeningOutput.numSpotsUsed = xsDataStatisticsIndexing.spotsUsed
xsDataISPyBScreeningOutput.numSpotsRejected = XSDataInteger(
iSpotsTotal - iSpotsUsed)
xsDataCrystal = xsDataIndexingSolutionSelected.crystal
xsDataISPyBScreeningOutput.mosaicityEstimated = XSDataBoolean(
False)
if (xsDataCrystal is not None):
if xsDataCrystal.mosaicity is not None:
xsDataISPyBScreeningOutput.mosaicity = xsDataCrystal.mosaicity
xsDataISPyBScreeningOutput.mosaicityEstimated = XSDataBoolean(
True)
xsDataCell = xsDataCrystal.cell
if xsDataCell is not None:
xsDataISPyBScreeningOutputLattice.unitCell_a = xsDataCell.length_a
xsDataISPyBScreeningOutputLattice.unitCell_b = xsDataCell.length_b
xsDataISPyBScreeningOutputLattice.unitCell_c = xsDataCell.length_c
xsDataISPyBScreeningOutputLattice.unitCell_alpha = xsDataCell.angle_alpha
xsDataISPyBScreeningOutputLattice.unitCell_beta = xsDataCell.angle_beta
xsDataISPyBScreeningOutputLattice.unitCell_gamma = xsDataCell.angle_gamma
xsDataSpaceGroup = xsDataCrystal.spaceGroup
if xsDataSpaceGroup is not None:
xsDataISPyBScreeningOutputLattice.spaceGroup = xsDataSpaceGroup.name
if (bSuccessfulIndexing):
xsDataISPyBScreeningOutput.setIndexingSuccess(XSDataBoolean(True))
if _strStatusMessage:
xsDataISPyBScreeningOutput.setStatusDescription(
XSDataString(_strStatusMessage))
else:
xsDataISPyBScreeningOutput.setStatusDescription(
XSDataString("Indexing successful"))
else:
xsDataISPyBScreeningOutput.setIndexingSuccess(XSDataBoolean(False))
if _strStatusMessage:
xsDataISPyBScreeningOutput.setStatusDescription(
XSDataString(_strStatusMessage))
else:
xsDataISPyBScreeningOutput.setStatusDescription(
XSDataString("Indexing failed"))
# Strategy information
xsDataResultStrategy = _xsDataResultCharacterisation.strategyResult
xsDataISPyBScreeningStrategyContainer = None
xsDataISPyBScreeningStrategyWedgeContainer = None
if xsDataResultStrategy is None:
xsDataISPyBScreeningOutput.setStrategySuccess(XSDataBoolean(False))
else:
if xsDataResultStrategy.collectionPlan == []:
xsDataISPyBScreeningOutput.setStrategySuccess(
XSDataBoolean(False))
else:
xsDataISPyBScreeningOutput.setStrategySuccess(
XSDataBoolean(True))
xsDataISPyBScreeningStrategyContainer = XSDataISPyBScreeningStrategyContainer(
)
xsDataISPyBScreeningStrategy = XSDataISPyBScreeningStrategy()
listXSDataCollectionPlan = xsDataResultStrategy.collectionPlan
fTotalExposureTime = 0.0
iTotalNumberOfImages = 0
fTotalRotationRange = 0.0
for xsDataCollectionPlan in listXSDataCollectionPlan:
numberOfImagesWedge = None
xsDataISPyBScreeningStrategyWedge = XSDataISPyBScreeningStrategyWedge(
)
xsDataISPyBScreeningStrategyWedgeContainer = XSDataISPyBScreeningStrategyWedgeContainer(
)
xsDataISPyBScreeningStrategyWedge.wedgeNumber = xsDataCollectionPlan.collectionPlanNumber
fWavelength = _xsDataResultCharacterisation.dataCollection.subWedge[
0].experimentalCondition.beam.wavelength.value
xsDataISPyBScreeningStrategyWedge.wavelength = XSDataDouble(
fWavelength)
if _fKappa is not None:
xsDataISPyBScreeningStrategyWedge.kappa = XSDataDouble(
_fKappa)
if _fPhi is not None:
xsDataISPyBScreeningStrategyWedge.phi = XSDataDouble(
_fPhi)
if (xsDataCollectionPlan.getComment() is not None):
strCollectionPlanComment = xsDataCollectionPlan.getComment(
).getValue()
xsDataISPyBScreeningStrategyWedge.comments = strCollectionPlanComment
xsDataStrategySummary = xsDataCollectionPlan.strategySummary
if xsDataStrategySummary is not None:
xsDataISPyBScreeningStrategyWedge.completeness = xsDataStrategySummary.completeness
xsDataISPyBScreeningStrategyWedge.resolution = xsDataStrategySummary.resolution
xsDataISPyBScreeningStrategyWedge.multiplicity = xsDataStrategySummary.redundancy
xsDataISPyBScreeningOutput.rankingResolution = xsDataStrategySummary.rankingResolution
if xsDataStrategySummary.totalExposureTime is not None:
fTotalExposureTime += xsDataStrategySummary.totalExposureTime.value
xsDataCollectionStrategy = xsDataCollectionPlan.collectionStrategy
if xsDataCollectionStrategy is not None:
for xsDataSubWedge in xsDataCollectionStrategy.subWedge:
numberOfImagesSubWedge = None
xsDataISPyBScreeningStrategySubWedge = XSDataISPyBScreeningStrategySubWedge(
)
xsDataISPyBScreeningStrategySubWedge.subWedgeNumber = xsDataSubWedge.subWedgeNumber
xsDataISPyBScreeningStrategySubWedge.axisStart = xsDataSubWedge.experimentalCondition.goniostat.rotationAxisStart
xsDataISPyBScreeningStrategySubWedge.axisEnd = xsDataSubWedge.experimentalCondition.goniostat.rotationAxisEnd
xsDataISPyBScreeningStrategySubWedge.oscillationRange = xsDataSubWedge.experimentalCondition.goniostat.oscillationWidth
# Number of images
if (xsDataSubWedge.experimentalCondition.goniostat.rotationAxisStart is not None) and \
(xsDataSubWedge.experimentalCondition.goniostat.rotationAxisEnd is not None) and \
(xsDataSubWedge.experimentalCondition.goniostat.oscillationWidth is not None):
numberOfImagesSubWedge = int((xsDataSubWedge.experimentalCondition.goniostat.rotationAxisEnd.value - \
xsDataSubWedge.experimentalCondition.goniostat.rotationAxisStart.value) / \
xsDataSubWedge.experimentalCondition.goniostat.oscillationWidth.value + 0.5)
xsDataISPyBScreeningStrategySubWedge.numberOfImages = XSDataInteger(
numberOfImagesSubWedge)
if numberOfImagesWedge is None:
numberOfImagesWedge = numberOfImagesSubWedge
else:
numberOfImagesWedge += numberOfImagesSubWedge
fTotalRotationRange += xsDataSubWedge.experimentalCondition.goniostat.oscillationWidth.value * numberOfImagesSubWedge
xsDataISPyBScreeningStrategySubWedge.exposureTime = xsDataSubWedge.experimentalCondition.beam.exposureTime
xsDataISPyBScreeningStrategySubWedge.transmission = xsDataSubWedge.experimentalCondition.beam.transmission
xsDataISPyBScreeningStrategyWedgeContainer.addScreeningStrategySubWedge(
xsDataISPyBScreeningStrategySubWedge)
if numberOfImagesWedge is not None:
xsDataISPyBScreeningStrategyWedge.numberOfImages = XSDataInteger(
numberOfImagesWedge)
iTotalNumberOfImages += numberOfImagesWedge
xsDataISPyBScreeningStrategyWedgeContainer.screeningStrategyWedge = xsDataISPyBScreeningStrategyWedge
xsDataISPyBScreeningStrategyContainer.addScreeningStrategyWedgeContainer(
xsDataISPyBScreeningStrategyWedgeContainer)
xsDataISPyBScreeningOutput.totalExposureTime = XSDataDouble(
fTotalExposureTime)
xsDataISPyBScreeningOutput.totalNumberOfImages = XSDataInteger(
iTotalNumberOfImages)
xsDataISPyBScreeningOutput.totalRotationRange = XSDataDouble(
fTotalRotationRange)
xsDataISPyBScreeningStrategyContainer.screeningStrategy = xsDataISPyBScreeningStrategy
# Assembly
xsDataISPyBScreeningOutputContainer = XSDataISPyBScreeningOutputContainer(
)
xsDataISPyBScreeningOutputContainer.screeningOutput = xsDataISPyBScreeningOutput
if xsDataISPyBScreeningOutputLattice is not None:
xsDataISPyBScreeningOutputContainer.addScreeningOutputLattice(
xsDataISPyBScreeningOutputLattice)
if xsDataISPyBScreeningStrategyContainer is not None:
xsDataISPyBScreeningOutputContainer.addScreeningStrategyContainer(
xsDataISPyBScreeningStrategyContainer)
return xsDataISPyBScreeningOutputContainer
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlSaxsModelingv1_0.process")
xsDataInputDammif = XSDataInputDammif(gnomOutputFile=self.xsGnomFile,
unit=XSDataString(self.unit),
symmetry=XSDataString(
self.symmetry),
mode=XSDataString(self.mode))
for i in range(self.dammif_jobs):
dammif = self.loadPlugin(self.strPluginExecDammif)
dammif.connectSUCCESS(self.doSuccessExecDammif)
dammif.connectFAILURE(self.doFailureExecDammif)
xsd = xsDataInputDammif.copyViaDict()
xsd.order = XSDataInteger(i + 1)
dammif.dataInput = xsd
self.addPluginToActionCluster(dammif)
self.dammif_plugins.append(dammif)
self.executeActionCluster()
self.synchronizeActionCluster()
for plugin in self.dammif_plugins:
if plugin.isFailure():
self.ERROR("dammif plugin %s-%08i failed" %
(plugin.getName(), plugin.getId()))
self.setFailure()
self.retrieveMessages(plugin)
if self.isFailure():
return
# retrieve results from best dammif
self.dammif = self.bestDammif()
self.chi2plot("chi2_R.png")
self.result.chiRfactorPlot = XSDataFile(
XSDataString(os.path.join(self.getWorkingDirectory(),
"chi2_R.png")))
# temporary results: use best dammif
self.result.fitFile = self.dammif.dataOutput.fitFile
self.result.logFile = self.dammif.dataOutput.logFile
self.result.pdbMoleculeFile = self.dammif.dataOutput.pdbMoleculeFile
self.result.pdbSolventFile = self.dammif.dataOutput.pdbSolventFile
# prepare an action cluster with all supcomb plugins
self.actclust_supcomb = EDActionCluster(self.cluster_size)
for idx in range(self.dammif_jobs):
if self.valid[idx]:
for ser in range(idx):
if self.valid[ser]:
supcomb = self.loadPlugin(self.strPluginExecSupcomb)
supcomb.dataInput = XSDataInputSupcomb(
templateFile=self.dammif_plugins[idx].dataOutput.
pdbMoleculeFile,
superimposeFile=self.dammif_plugins[ser].
dataOutput.pdbMoleculeFile,
name=self.dammif_plugins[ser].dataOutput.model.name
)
self.supcomb_plugins[(idx, ser)] = supcomb
self.actclust_supcomb.addAction(supcomb)
self.actclust_supcomb.executeSynchronous()
for key, plugin in self.supcomb_plugins.items():
if plugin.isFailure():
self.ERROR(
"supcomb plugin for model pair (%i,%i) %s-%08i failed" %
(key[0] + 1, key[1] + 1, plugin.getName(), plugin.getId()))
self.setFailure()
self.retrieveMessages(plugin)
if self.isFailure():
return
self.makeNSDarray("nsd.png")
self.result.nsdPlot = XSDataFile(
XSDataString(os.path.join(self.getWorkingDirectory(), "nsd.png")))
idx = self.ref
self.actclust_supcomb = EDActionCluster(self.cluster_size)
for ser in range(self.ref + 1, self.dammif_jobs):
if self.valid[ser]:
supcomb = self.loadPlugin(self.strPluginExecSupcomb)
supcomb.dataInput = XSDataInputSupcomb(
templateFile=self.dammif_plugins[
self.ref].dataOutput.pdbMoleculeFile,
superimposeFile=self.dammif_plugins[ser].dataOutput.
pdbMoleculeFile,
name=self.dammif_plugins[ser].dataOutput.model.name)
self.supcomb_plugins[(self.ref, ser)] = supcomb
self.actclust_supcomb.addAction(supcomb)
self.actclust_supcomb.executeSynchronous()
for ser in range(self.ref + 1, self.dammif_jobs):
if self.valid[ser]:
plugin = self.supcomb_plugins[(self.ref, ser)]
if plugin.isFailure():
self.ERROR(
"supcomb plugin for model pair (%i,%i) %s-%08i failed"
% (self.ref + 1, ser + 1, plugin.getName(),
plugin.getId()))
self.setFailure()
self.retrieveMessages(plugin)
if self.isFailure():
return
for i in range(self.dammif_jobs):
if i == self.ref or not self.valid[i]:
model = self.dammif_plugins[i].dataOutput.model
else:
model = self.supcomb_plugins[(self.ref, i)].dataOutput.model
model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt
# model.chiSqrt = self.dammif_plugins[i].dataOutput.model.chiSqrt
self.symlink(model.pdbFile.path.value, "model-%02i.pdb" % (i + 1))
self.result.dammifModels[i] = model
# Now that all (valid) models are aligned we can combine them using damaver
pdbFiles = [self.dammif_plugins[self.ref].dataOutput.pdbMoleculeFile]
for idx in range(self.dammif_jobs):
if self.valid[idx] and idx != self.ref:
pdbFiles.append(
self.supcomb_plugins[(self.ref,
idx)].dataOutput.outputFilename)
damaver = self.loadPlugin(self.strPluginExecDamaver)
damaver.dataInput = XSDataInputDamaver(pdbInputFiles=pdbFiles,
automatic=XSDataBoolean(False))
damaver.connectSUCCESS(self.doSuccessExecDamaver)
damaver.connectFAILURE(self.doFailureExecDamaver)
damaver.executeSynchronous()
if self.isFailure():
return
damfilt = self.loadPlugin(self.strPluginExecDamfilt)
damfilt.dataInput = XSDataInputDamfilt(
inputPdbFile=damaver.dataOutput.damaverPdbFile)
damfilt.connectSUCCESS(self.doSuccessExecDamfilt)
damfilt.connectFAILURE(self.doFailureExecDamfilt)
damfilt.execute()
########################################################################
# TODO: This is a dead end : do it in parallel
########################################################################
if self.isFailure():
return
damstart = self.loadPlugin(self.strPluginExecDamstart)
damstart.dataInput = XSDataInputDamstart(
inputPdbFile=damaver.dataOutput.damaverPdbFile)
damstart.connectSUCCESS(self.doSuccessExecDamstart)
damstart.connectFAILURE(self.doFailureExecDamstart)
damstart.executeSynchronous()
if self.isFailure():
return
########################################################################
# Finally call dammin
########################################################################
if self.config.get("do_dammin") in ["False", "0", False, 0]:
return
dammin = self.loadPlugin(self.strPluginExecDammin)
dammin.dataInput = XSDataInputDammin(
pdbInputFile=damstart.dataOutput.outputPdbFile,
gnomOutputFile=self.xsGnomFile,
symmetry=XSDataString(self.symmetry),
mode=XSDataString(self.mode))
dammin.connectSUCCESS(self.doSuccessExecDammin)
dammin.connectFAILURE(self.doFailureExecDammin)
dammin.executeSynchronous()
def from_params(self, data_collection, char_params):
edna_input = XSDataInputMXCuBE.parseString(self.edna_default_input)
if data_collection.id:
edna_input.setDataCollectionId(XSDataInteger(data_collection.id))
#Beam object
beam = edna_input.getExperimentalCondition().getBeam()
try:
transmission = self.collect_obj.get_transmission()
beam.setTransmission(XSDataDouble(transmission))
except AttributeError:
pass
try:
wavelength = self.collect_obj.get_wavelength()
beam.setWavelength(XSDataWavelength(wavelength))
except AttributeError:
pass
try:
beam.setFlux(XSDataFlux(self.collect_obj.get_measured_intensity()))
except AttributeError:
pass
try:
beamsize = self.get_beam_size()
if not None in beamsize:
beam.setSize(
XSDataSize(x=XSDataLength(float(beamsize[0])),
y=XSDataLength(float(beamsize[1]))))
except AttributeError:
pass
#Optimization parameters
diff_plan = edna_input.getDiffractionPlan()
aimed_i_sigma = XSDataDouble(char_params.aimed_i_sigma)
aimed_completness = XSDataDouble(char_params.aimed_completness)
aimed_multiplicity = XSDataDouble(char_params.aimed_multiplicity)
aimed_resolution = XSDataDouble(char_params.aimed_resolution)
complexity = char_params.strategy_complexity
complexity = XSDataString(qme.STRATEGY_COMPLEXITY[complexity])
permitted_phi_start = XSDataAngle(char_params.permitted_phi_start)
_range = char_params.permitted_phi_end - char_params.permitted_phi_start
rotation_range = XSDataAngle(_range)
diff_plan.setAimedIOverSigmaAtHighestResolution(aimed_i_sigma)
diff_plan.setAimedCompleteness(aimed_completness)
if char_params.use_aimed_multiplicity:
diff_plan.setAimedMultiplicity(aimed_multiplicity)
if char_params.use_aimed_resolution:
diff_plan.setAimedResolution(aimed_resolution)
diff_plan.setComplexity(complexity)
if char_params.use_permitted_rotation:
diff_plan.setUserDefinedRotationStart(permitted_phi_start)
diff_plan.setUserDefinedRotationRange(rotation_range)
#Vertical crystal dimension
sample = edna_input.getSample()
sample.getSize().setY(XSDataLength(char_params.max_crystal_vdim))
sample.getSize().setZ(XSDataLength(char_params.min_crystal_vdim))
#Radiation damage model
sample.setSusceptibility(XSDataDouble(char_params.rad_suscept))
sample.setChemicalComposition(None)
sample.setRadiationDamageModelBeta(XSDataDouble(char_params.beta /
1e6))
sample.setRadiationDamageModelGamma(
XSDataDouble(char_params.gamma / 1e6))
diff_plan.setForcedSpaceGroup(XSDataString(char_params.space_group))
# Characterisation type - Routine DC
if char_params.use_min_dose:
pass
if char_params.use_min_time:
time = XSDataTime(char_params.min_time)
diff_plan.setMaxExposureTimePerDataCollection(time)
# Account for radiation damage
if char_params.induce_burn:
self.modify_strategy_option(diff_plan, "-DamPar")
# Characterisation type - SAD
if char_params.opt_sad:
if char_params.auto_res:
diff_plan.setAnomalousData(XSDataBoolean(True))
else:
diff_plan.setAnomalousData(XSDataBoolean(False))
self.modify_strategy_option(diff_plan, "-SAD yes")
diff_plan.setAimedResolution(XSDataDouble(char_params.sad_res))
else:
diff_plan.setAnomalousData(XSDataBoolean(False))
#Data set
data_set = XSDataMXCuBEDataSet()
acquisition_parameters = data_collection.acquisitions[
0].acquisition_parameters
path_template = data_collection.acquisitions[0].path_template
path_str = os.path.join(path_template.directory,
path_template.get_image_file_name())
for img_num in range(int(acquisition_parameters.num_images)):
image_file = XSDataFile()
path = XSDataString()
path.setValue(path_str % (img_num + 1))
image_file.setPath(path)
data_set.addImageFile(image_file)
edna_input.addDataSet(data_set)
edna_input.process_directory = path_template.process_directory
return edna_input
def create_processing_input(self, data_collection):
"""Creates dozor input file base on data collection parameters
:param data_collection: data collection object
:type : queue_model_objects.DataCollection
"""
acquisition = data_collection.acquisitions[0]
acq_params = acquisition.acquisition_parameters
input_file = XSDataInputControlDozor()
image_file_template = "%s_%%d_%%05d.cbf" % (
acquisition.path_template.get_prefix())
if "p14" in acquisition.path_template.directory:
template = os.path.join(acquisition.path_template.directory.replace("dataInt/p14","ramdisk"),
image_file_template)
else:
template = os.path.join(acquisition.path_template.directory,
image_file_template)
first_image_num = acq_params.first_image
images_num = acq_params.num_images
last_image_num = first_image_num + images_num - 1
run_number = acquisition.path_template.run_number
lines_num = acq_params.num_lines
pixel_min = 0
pixel_max = 0
beamstop_size = 0
beamstop_distance = 0
beamstop_direction = 0
pixel_min = self.detector_hwobj.get_pixel_min()
pixel_max = self.detector_hwobj.get_pixel_max()
beamstop_size = self.beamstop_hwobj.get_size()
beamstop_distance = self.beamstop_hwobj.get_distance()
beamstop_direction = self.beamstop_hwobj.get_direction()
if data_collection.grid:
grid_params = data_collection.grid.get_properties()
reversing_rotation = grid_params["reversing_rotation"]
else:
reversing_rotation = False
self.params_dict["template"] = template
self.params_dict["first_image_num"] = first_image_num
self.params_dict["images_num"] = images_num
self.params_dict["lines_num"] = lines_num
self.params_dict["images_per_line"] = images_num / lines_num
self.params_dict["run_number"] = run_number
self.params_dict["osc_midle"] = acq_params.osc_start
self.params_dict["osc_range"] = acq_params.osc_range
self.params_dict["pixel_min"] = pixel_min
self.params_dict["pixel_max"] = pixel_max
self.params_dict["beamstop_size"] = beamstop_size
self.params_dict["beamstop_distance"] = beamstop_distance
self.params_dict["beamstop_direction"] = beamstop_direction
self.params_dict["status"] = "Started"
self.params_dict["title"] = "%s_%d_#####.cbf (%d - %d)" % \
(acquisition.path_template.get_prefix(),
run_number,
first_image_num,
last_image_num)
self.params_dict["comments"] = "Scan lines: %d, frames per line: %d" % \
(lines_num, images_num / lines_num)
if lines_num > 1:
self.params_dict["dx_mm"] = grid_params["dx_mm"]
self.params_dict["dy_mm"] = grid_params["dy_mm"]
self.params_dict["steps_x"] = grid_params["steps_x"]
self.params_dict["steps_y"] = grid_params["steps_y"]
self.params_dict["xOffset"] = grid_params["xOffset"]
self.params_dict["yOffset"] = grid_params["yOffset"]
else:
self.params_dict["steps_y"] = 1
input_file.setTemplate(XSDataString(template))
input_file.setFirst_image_number(XSDataInteger(first_image_num))
input_file.setLast_image_number(XSDataInteger(last_image_num))
input_file.setFirst_run_number(XSDataInteger(run_number))
input_file.setLast_run_number(XSDataInteger(run_number))
input_file.setLine_number_of(XSDataInteger(lines_num))
input_file.setReversing_rotation(XSDataBoolean(reversing_rotation))
input_file.setPixelMin(XSDataInteger(pixel_min))
input_file.setPixelMax(XSDataInteger(pixel_max))
input_file.setBeamstopSize(XSDataDouble(beamstop_size))
input_file.setBeamstopDistance(XSDataDouble(beamstop_distance))
input_file.setBeamstopDirection(XSDataString(beamstop_direction))
return input_file
def process(self, _edObject=None):
"""
Uses ToolsForCollectionWebService for storing the workflow status
"""
EDPluginExec.process(self)
self.DEBUG("EDPluginISPyBGetSampleInformationv1_4.process")
# First get the image ID
xsDataInputGetSampleInformation = self.getDataInput()
httpAuthenticatedToolsForCollectionWebService = HttpAuthenticated(username=self.strUserName, password=self.strPassWord)
clientToolsForBLSampleWebServiceWsdl = Client(self.strToolsForBLSampleWebServiceWsdl, transport=httpAuthenticatedToolsForCollectionWebService)
iSampleId = self.getXSValue(xsDataInputGetSampleInformation.sampleId)
sampleInfo = clientToolsForBLSampleWebServiceWsdl.service.getSampleInformation(iSampleId)
self.DEBUG("Sample info from ISPyB: %r" % sampleInfo)
if sampleInfo is not None:
if "code" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.code = XSDataString(sampleInfo.code)
if "cellA" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.cellA = XSDataDouble(sampleInfo.cellA)
if "cellB" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.cellB = XSDataDouble(sampleInfo.cellB)
if "cellC" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.cellC = XSDataDouble(sampleInfo.cellC)
if "cellAlpha" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.cellAlpha = XSDataDouble(sampleInfo.cellAlpha)
if "cellBeta" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.cellBeta = XSDataDouble(sampleInfo.cellBeta)
if "cellGamma" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.cellGamma = XSDataDouble(sampleInfo.cellGamma)
if "containerSampleChangerLocation" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.containerSampleChangerLocation = XSDataString(sampleInfo.containerSampleChangerLocation)
if "crystalSpaceGroup" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.crystalSpaceGroup = XSDataString(sampleInfo.crystalSpaceGroup)
if "experimentType" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.experimentType = XSDataString(sampleInfo.experimentType)
if "holderLength" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.holderLength = XSDataDouble(sampleInfo.holderLength)
if "minimalResolution" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.minimalResolution = XSDataDouble(sampleInfo.minimalResolution)
if "proteinAcronym" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.proteinAcronym = XSDataString(sampleInfo.proteinAcronym)
if "sampleId" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.sampleId = XSDataInteger(sampleInfo.sampleId)
if "sampleLocation" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.sampleLocation = XSDataString(sampleInfo.sampleLocation)
if "sampleName" in sampleInfo:
self.xsDataResultISPyBGetSampleInformation.sampleName = XSDataString(sampleInfo.sampleName)
if "diffractionPlan" in sampleInfo:
xsDataISPyBDiffractionPlan = XSDataISPyBDiffractionPlan()
if "aimedCompleteness" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.aimedCompleteness = XSDataDouble(sampleInfo.diffractionPlan.aimedCompleteness)
if "aimedIOverSigmaAtHighestResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.aimedIOverSigmaAtHighestResolution = XSDataDouble(sampleInfo.diffractionPlan.aimedIOverSigmaAtHighestResolution)
if "aimedMultiplicity" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.aimedMultiplicity = XSDataDouble(sampleInfo.diffractionPlan.aimedMultiplicity)
if "aimedResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.aimedResolution = XSDataDouble(sampleInfo.diffractionPlan.aimedResolution)
if "anomalousData" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.anomalousData = XSDataBoolean(sampleInfo.diffractionPlan.anomalousData)
if "comments" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.comments = XSDataString(sampleInfo.diffractionPlan.comments)
if "complexity" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.complexity = XSDataString(sampleInfo.diffractionPlan.complexity)
if "diffractionPlanId" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.diffractionPlanId = XSDataInteger(sampleInfo.diffractionPlan.diffractionPlanId)
if "estimateRadiationDamage" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.estimateRadiationDamage = XSDataBoolean(sampleInfo.diffractionPlan.estimateRadiationDamage)
if "experimentKind" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.experimentKind = XSDataString(sampleInfo.diffractionPlan.experimentKind)
if "exposureTime" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.exposureTime = XSDataDouble(sampleInfo.diffractionPlan.exposureTime)
if "forcedSpaceGroup" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.forcedSpaceGroup = XSDataString(sampleInfo.diffractionPlan.forcedSpaceGroup)
if "kappaStrategyOption" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.kappaStrategyOption = XSDataString(sampleInfo.diffractionPlan.kappaStrategyOption)
if "maxDimAccrossSpindleAxis" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.maxDimAccrossSpindleAxis = XSDataDouble(sampleInfo.diffractionPlan.maxDimAccrossSpindleAxis)
if "maximalResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.maximalResolution = XSDataDouble(sampleInfo.diffractionPlan.maximalResolution)
if "minDimAccrossSpindleAxis" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.minDimAccrossSpindleAxis = XSDataDouble(sampleInfo.diffractionPlan.minDimAccrossSpindleAxis)
if "minimalResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.minimalResolution = XSDataDouble(sampleInfo.diffractionPlan.minimalResolution)
if "numberOfPositions" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.numberOfPositions = XSDataInteger(sampleInfo.diffractionPlan.numberOfPositions)
if "observedResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.observedResolution = XSDataDouble(sampleInfo.diffractionPlan.observedResolution)
if "oscillationRange" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.oscillationRange = XSDataDouble(sampleInfo.diffractionPlan.oscillationRange)
if "preferredBeamSizeX" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.preferredBeamSizeX = XSDataDouble(sampleInfo.diffractionPlan.preferredBeamSizeX)
if "preferredBeamSizeY" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.preferredBeamSizeY = XSDataDouble(sampleInfo.diffractionPlan.preferredBeamSizeY)
if "radiationSensitivity" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.radiationSensitivity = XSDataDouble(sampleInfo.diffractionPlan.radiationSensitivity)
if "radiationSensitivityBeta" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.radiationSensitivityBeta = XSDataDouble(sampleInfo.diffractionPlan.radiationSensitivityBeta)
if "radiationSensitivityGamma" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.radiationSensitivityGamma = XSDataDouble(sampleInfo.diffractionPlan.radiationSensitivityGamma)
if "requiredCompleteness" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.requiredCompleteness = XSDataDouble(sampleInfo.diffractionPlan.requiredCompleteness)
if "requiredMultiplicity" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.requiredMultiplicity = XSDataDouble(sampleInfo.diffractionPlan.requiredMultiplicity)
if "requiredResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.requiredResolution = XSDataDouble(sampleInfo.diffractionPlan.requiredResolution)
if "screeningResolution" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.screeningResolution = XSDataDouble(sampleInfo.diffractionPlan.screeningResolution)
if "anomalousScatterer" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.anomalousScatterer = XSDataString(sampleInfo.diffractionPlan.anomalousScatterer)
if "strategyOption" in sampleInfo.diffractionPlan:
xsDataISPyBDiffractionPlan.strategyOption = XSDataString(sampleInfo.diffractionPlan.strategyOption)
self.xsDataResultISPyBGetSampleInformation.diffractionPlan = xsDataISPyBDiffractionPlan
self.DEBUG("EDPluginISPyBGetSampleInformationv1_4.process: result=%s" % pprint.pformat(self.xsDataResultISPyBGetSampleInformation))
def create_processing_input(self, data_collection, processing_params,
grid_object):
"""
Descript. : Creates dozor input file base on data collection parameters
Args. : data_collection (object)
Return. : processing_input_file (object)
"""
acquisition = data_collection.acquisitions[0]
acq_params = acquisition.acquisition_parameters
processing_input_file = XSDataInputControlDozor()
_run = "_%d_" % acquisition.path_template.run_number
image_file_template = "%s_%%d_%%05d.cbf" % (
acquisition.path_template.get_prefix())
template = os.path.join(acquisition.path_template.directory,
image_file_template)
first_image_num = acq_params.first_image
images_num = acq_params.num_images
last_image_num = first_image_num + images_num - 1
run_number = acquisition.path_template.run_number
lines_num = acq_params.num_lines
pixel_min = 0
pixel_max = 0
beamstop_size = 0
beamstop_distance = 0
beamstop_direction = 0
try:
pixel_min = self.detector_hwobj.get_pixel_min()
pixel_max = self.detector_hwobj.get_pixel_max()
except:
pass
try:
beamstop_size = self.beamstop_hwobj.get_beamstop_size()
beamstop_distance = self.beamstop_hwobj.get_beamstop_distance()
beamstop_direction = self.beamstop_hwobj.get_beamstop_direction()
except:
pass
grid_params = grid_object.get_properties()
reversing_rotation = grid_params["reversing_rotation"]
processing_params["template"] = template
processing_params["first_image_num"] = first_image_num
processing_params["images_num"] = images_num
processing_params["lines_num"] = lines_num
processing_params["images_per_line"] = images_num / lines_num
processing_params["run_number"] = run_number
processing_params["pixel_min"] = pixel_min
processing_params["pixel_max"] = pixel_max
processing_params["beamstop_size"] = beamstop_size
processing_params["beamstop_distance"] = beamstop_distance
processing_params["beamstop_direction"] = beamstop_direction
processing_params["status"] = "Started"
processing_params["title"] = "%s_%d_xxxxx.cbf (%d - %d)" % \
(acquisition.path_template.get_prefix(),
acquisition.path_template.run_number,
first_image_num,
last_image_num)
processing_params["comments"] = "Scan lines: %d, frames per line: %d" % \
(lines_num, images_num / lines_num )
if lines_num > 1:
processing_params["dx_mm"] = grid_params["dx_mm"]
processing_params["dy_mm"] = grid_params["dy_mm"]
processing_params["steps_x"] = grid_params["steps_x"]
processing_params["steps_y"] = grid_params["steps_y"]
processing_params["xOffset"] = grid_params["xOffset"]
processing_params["yOffset"] = grid_params["yOffset"]
processing_input_file.setTemplate(XSDataString(template))
processing_input_file.setFirst_image_number(
XSDataInteger(first_image_num))
processing_input_file.setLast_image_number(
XSDataInteger(last_image_num))
processing_input_file.setFirst_run_number(XSDataInteger(run_number))
processing_input_file.setLast_run_number(XSDataInteger(run_number))
processing_input_file.setLine_number_of(XSDataInteger(lines_num))
processing_input_file.setReversing_rotation(
XSDataBoolean(reversing_rotation))
processing_input_file.setPixelMin(
XSDataInteger(pixel_min)) # should be -1 for real data
processing_input_file.setPixelMax(XSDataInteger(pixel_max))
processing_input_file.setBeamstopSize(XSDataDouble(beamstop_size))
processing_input_file.setBeamstopDistance(
XSDataDouble(beamstop_distance))
processing_input_file.setBeamstopDirection(
XSDataString(beamstop_direction))
return processing_input_file, processing_params
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginXDSGeneratev1_0.preProcess")
if self.dataInput.doAnomAndNonanom is not None:
if self.dataInput.doAnomAndNonanom.value:
self.doAnomAndNonanom = True
else:
self.doAnomAndNonanom = False
self.xds_anom = self.loadPlugin('EDPluginExecMinimalXdsv1_0')
if self.doAnomAndNonanom:
self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXdsv1_0')
path = os.path.abspath(self.dataInput.previous_run_dir.value)
# The MinimalXds plugin takes care of creating determining the
# real images directory and creating a symlink to it so we
# only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
# to not be relative anymore. We'll copy it to our own dir
# beforehand to avoid clobbering it
xdsinp = os.path.join(path, 'XDS.INP')
new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
copyfile(xdsinp, new_xdsinp)
parsed_config = parse_xds_file(new_xdsinp)
file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(
os.path.join(path, file_template))
dump_xds_file(new_xdsinp, parsed_config)
# create the data inputs now we know the files are here
input_anom = XSDataMinimalXdsIn()
input_anom.input_file = XSDataString(new_xdsinp)
input_anom.friedels_law = XSDataBoolean(False)
input_anom.job = XSDataString('CORRECT')
input_anom.resolution = self.dataInput.resolution
input_anom.resolution_range = [
XSDataDouble(60), self.dataInput.resolution
]
input_anom.spacegroup = self.dataInput.spacegroup
input_anom.unit_cell = self.dataInput.unit_cell
self.xds_anom.dataInput = input_anom
xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
if self.doAnomAndNonanom:
input_noanom = XSDataMinimalXdsIn()
input_noanom.input_file = XSDataString(new_xdsinp)
input_noanom.friedels_law = XSDataBoolean(True)
input_noanom.job = XSDataString('CORRECT')
input_noanom.resolution_range = [
XSDataDouble(60), self.dataInput.resolution
]
input_noanom.spacegroup = self.dataInput.spacegroup
input_noanom.unit_cell = self.dataInput.unit_cell
self.xds_noanom.dataInput = input_noanom
xds_noanom_dir = os.path.abspath(
self.xds_noanom.getWorkingDirectory())
# let's make some links!
for f in self._to_link:
os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
if self.doAnomAndNonanom:
os.symlink(f, os.path.join(xds_noanom_dir,
os.path.basename(f)))
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
elif EDUtilsPath.isMAXIV():
template = ispybDataCollection.fileTemplate
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.fromN is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.fromN.value
if self.dataInput.toN is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.toN.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.path.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
if EDUtilsPath.isEMBL():
fileTemplate = template.replace("#####", "%05d")
else:
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
if EDUtilsPath.isALBA():
_processDirectory = "_".join(pathToStartImage.split('_')[:-1])
from datetime import datetime
_id = datetime.now().strftime('%Y%m%d_%H%M%S')
self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
else:
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"autoPROC", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# The resultsDirectory is not used at ALBA (only pyarchDirectory)
if EDUtilsPath.isALBA():
self.resultsDirectory = None
# Determine pyarch prefix
if EDUtilsPath.isALBA():
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
listPrefix[-2])
else:
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isMAXIV():
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
fWaitFileTimeout = 60 # s
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart)
self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart)
self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart)
# Prepare input to execution plugin
if self.doAnom:
xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCAnom.symm = self.dataInput.symm
xsDataInputAutoPROCAnom.cell = self.dataInput.cell
xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
if self.doNoanom:
xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
xsDataAutoPROCIdentifier.templateN = XSDataString(template)
xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
if self.doAnom:
xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
if self.doNoanom:
xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
if self.doNoanom:
xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
timeStart = time.localtime()
if self.doAnom:
self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
self.edPluginExecAutoPROCAnom.execute()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
self.edPluginExecAutoPROCNoanom.execute()
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
if self.doNoanom:
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def makeXML(self, filename):
"""Here we create the XML string to be passed to the EDNA plugin from the input filename
This can / should be modified by the final user
@param filename: full path of the input file
@type filename: python string representing the path
@rtype: XML string
@return: python string
"""
self.header = None
dirname, basename = os.path.split(filename)
if not basename.startswith(self.prefix):
return
if self.normalizedSuffix and basename.endswith(self.normalizedSuffix):
return
if basename.startswith(self.flatPrefix):
return
if not basename.endswith(self.suffix):
return
if self.fScaleData:
fScaleData = self.fScaleData
else:
fScaleData = self.getNbFrames(filename)
if not fScaleData:
fScaleData = 1.0
if self.fScaleDark:
fScaleDark = self.fScaleDark
else:
fScaleDark = self.getNbFrames(self.darks[0]["path"])
if not fScaleDark:
fScaleDark = 1.0
if isinstance(self.reference, int):
reference = self.reference
else:
reference = -1
xsd = XSDataInputFullFieldXAS(
HDF5File=XSDataFile(path=XSDataString(self.HDF5)),
internalHDF5Path=XSDataString(self.internalHdf5),
measureOffset=self.getXSDMeasureOffset(),
dark=self.getXsdDark(),
reference=XSDataInteger(reference),
data=[
XSDataImageExt(path=XSDataString(filename),
exposureTime=XSDataTime(
self.getExposureTime(filename)))
],
dataScaleFactor=XSDataDouble(fScaleData),
darkScaleFactor=XSDataDouble(fScaleDark),
)
if self.dontAlign:
xsd.dontAlign = XSDataBoolean(self.dontAlign)
extendedPrefix = ""
number = ""
started = False
if self.lstSubscanSize:
subScanDigit = []
for i in basename[len(self.prefix):]:
if started and i == "_":
if len(number) > 0:
subScanDigit.append(number)
number = ""
continue
if started and not i.isdigit():
if number:
subScanDigit.append(number)
number = ""
break
if not started:
if i.isdigit():
started = True
else:
extendedPrefix += i
if started:
number += i
if not subScanDigit:
print("ERROR: no index guessed !!!")
return ""
elif len(subScanDigit) == 1:
index = int(subScanDigit[0])
else: # len(subScanDigit) > 1:
index = 0
for i in range(int(subScanDigit[0])):
index += self.lstSubscanSize[i]
index += int(subScanDigit[1])
extendedPrefix += "_".join(subScanDigit[:2])
else:
for i in basename[len(self.prefix):]:
extendedPrefix += i
if started and not i.isdigit():
break
if not started and i.isdigit():
started = True
if started:
number += i
index = int(number)
xsd.index = XSDataInteger(index)
if self.normalizedSuffix:
pr = os.path.splitext(os.path.abspath(filename))[0]
xsd.saveNormalized = XSDataFile(
path=XSDataString(pr + self.normalizedSuffix))
energy = self.getEnergy(filename)
if energy is not None:
xsd.energy = XSDataDouble(energy)
flatprefix = self.flatPrefix + extendedPrefix
listFlats = []
for oneFile in os.listdir(dirname):
if oneFile.startswith(flatprefix) and oneFile.endswith(
self.suffix):
oneCompleteFile = os.path.abspath(
os.path.join(dirname, oneFile))
xsdFileFlat1 = XSDataImageExt(
path=XSDataString(oneCompleteFile),
exposureTime=XSDataTime(
self.getExposureTime(oneCompleteFile)))
listFlats.append(xsdFileFlat1)
xsd.flat = listFlats
if len(listFlats) != 2:
EDVerbose.WARNING("Not exactly 2 flats were found:" +
" ".join([a.path.value for a in listFlats]))
EDVerbose.WARNING("Those are the flat prefix ans suffix: %s\t%s" %
(flatprefix, self.suffix))
if self.fScaleFlat:
fScaleFlat = self.fScaleFlat
else:
fScaleFlat = self.getNbFrames(oneCompleteFile)
if not fScaleFlat:
fScaleFlat = 1.0
xsd.flatScaleFactor = XSDataDouble(fScaleFlat)
return xsd.marshal()
def uploadToISPyB(self, edPluginExecAutoPROC, isAnom, isStaraniso, proposal, timeStart, timeEnd):
if isAnom:
anomString = "anom"
else:
anomString = "noanom"
if isStaraniso:
staranisoString = "_staraniso"
else:
staranisoString = ""
# Read the generated ISPyB xml file
pathToISPyBXML = None
if isStaraniso:
if edPluginExecAutoPROC.dataOutput.ispybXML_staraniso is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.ispybXML is not None:
pathToISPyBXML = edPluginExecAutoPROC.dataOutput.ispybXML.path.value
if pathToISPyBXML is not None:
autoProcContainer = AutoProcContainer.parseFile(pathToISPyBXML)
# "Fix" certain entries in the ISPyB xml file
autoProcScalingContainer = autoProcContainer.AutoProcScalingContainer
for autoProcScalingStatistics in autoProcScalingContainer.AutoProcScalingStatistics:
if isAnom:
autoProcScalingStatistics.anomalous = True
else:
autoProcScalingStatistics.anomalous = False
# Convert from fraction to %
autoProcScalingStatistics.rMerge *= 100.0
autoProcScalingStatistics.rMeasWithinIPlusIMinus *= 100
autoProcScalingStatistics.rMeasAllIPlusIMinus *= 100
autoProcScalingStatistics.rPimWithinIPlusIMinus *= 100
autoProcScalingStatistics.rPimAllIPlusIMinus *= 100
autoProcIntegrationContainer = autoProcScalingContainer.AutoProcIntegrationContainer
image = autoProcIntegrationContainer.Image
if self.dataInput.dataCollectionId is not None:
image.dataCollectionId = self.dataInput.dataCollectionId.value
autoProcIntegration = autoProcIntegrationContainer.AutoProcIntegration
autoProcProgramContainer = autoProcContainer.AutoProcProgramContainer
autoProcProgram = autoProcProgramContainer.AutoProcProgram
if isAnom:
autoProcIntegration.anomalous = True
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdAnom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdAnom
else:
autoProcIntegration.anomalous = False
if isStaraniso:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanomStaraniso
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanomStaraniso
else:
autoProcIntegration.autoProcIntegrationId = self.autoProcIntegrationIdNoanom
autoProcProgram.autoProcProgramId = self.autoProcProgramIdNoanom
if self.reprocess:
autoProcProgram.processingPrograms = "autoPROC" + staranisoString + " reprocess"
else:
autoProcProgram.processingPrograms = "autoPROC" + staranisoString
autoProcProgram.processingStartTime = time.strftime("%a %b %d %H:%M:%S %Y", timeStart)
autoProcProgram.processingEndTime = time.strftime("%a %b %d %H:%M:%S %Y", timeEnd)
autoProcProgram.processingStatus = "SUCCESS"
# EDNA-245 - remove "truncate_{early,late}-unique.mtz" from autoProcProgramContainer.AutoProcProgramAttachment
autoProcProgramContainer.AutoProcProgramAttachment[:] = [x for x in autoProcProgramContainer.AutoProcProgramAttachment if not self.matchesTruncateEarlyLate(x.fileName) ]
for autoProcProgramAttachment in autoProcProgramContainer.AutoProcProgramAttachment:
if autoProcProgramAttachment.fileName == "summary.html":
# Check if summary_inlined.html exists
summaryInlinedHtmlPath = os.path.join(autoProcProgramAttachment.filePath, "summary_inlined.html")
if os.path.exists(summaryInlinedHtmlPath):
summaryName = "summary_inlined"
summaryHtmlPath = summaryInlinedHtmlPath
else:
summaryName = "summary"
summaryHtmlPath = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
# Replace opidXX with user name
htmlSummary = open(summaryHtmlPath).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
htmlSummary = htmlSummary.replace(userString1, userString2)
open(summaryHtmlPath, "w").write(htmlSummary)
# Upload summary.html
pathtoFile = summaryHtmlPath
pyarchFile = self.pyarchPrefix + "_{0}_{1}.html".format(anomString, summaryName)
if not pyarchFile in self.listPyarchFile:
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
self.listPyarchFile.append(pyarchFile)
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = os.path.basename(pyarchFile)
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
if summaryName == "summary":
# Convert the summary.html to summary.pdf
xsDataInputHTML2PDF = XSDataInputHTML2PDF()
xsDataInputHTML2PDF.addHtmlFile(XSDataFile(XSDataString(summaryHtmlPath)))
xsDataInputHTML2PDF.paperSize = XSDataString("A3")
xsDataInputHTML2PDF.lowQuality = XSDataBoolean(True)
edPluginHTML2Pdf = self.loadPlugin("EDPluginHTML2PDFv1_0", "EDPluginHTML2PDFv1_0_{0}".format(anomString))
edPluginHTML2Pdf.dataInput = xsDataInputHTML2PDF
edPluginHTML2Pdf.executeSynchronous()
pdfFile = edPluginHTML2Pdf.dataOutput.pdfFile.path.value
pyarchPdfFile = self.pyarchPrefix + "_" + anomString + "_" + os.path.basename(pdfFile)
# Copy file to results directory and pyarch
if self.resultsDirectory:
shutil.copy(pdfFile, os.path.join(self.resultsDirectory, pyarchPdfFile))
if self.pyarchDirectory is not None:
shutil.copy(pdfFile, os.path.join(self.pyarchDirectory, pyarchPdfFile))
autoProcProgramAttachmentPdf = AutoProcProgramAttachment()
autoProcProgramAttachmentPdf.fileName = pyarchPdfFile
autoProcProgramAttachmentPdf.filePath = self.pyarchDirectory
autoProcProgramAttachmentPdf.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachmentPdf)
elif autoProcProgramAttachment.fileName == "truncate-unique.mtz":
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_{0}_truncate.mtz".format(anomString)
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
else:
pathtoFile = os.path.join(autoProcProgramAttachment.filePath, autoProcProgramAttachment.fileName)
pyarchFile = self.pyarchPrefix + "_" + anomString + "_" + autoProcProgramAttachment.fileName
if self.resultsDirectory:
shutil.copy(pathtoFile, os.path.join(self.resultsDirectory, pyarchFile))
if self.pyarchDirectory is not None:
shutil.copy(pathtoFile, os.path.join(self.pyarchDirectory, pyarchFile))
autoProcProgramAttachment.fileName = pyarchFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
# Add XSCALE.LP file if present
processDirectory = edPluginExecAutoPROC.dataOutput.processDirectory[0].path.value
pathToXSCALELog = os.path.join(processDirectory, "XSCALE.LP")
if os.path.exists(pathToXSCALELog):
pyarchXSCALELog = self.pyarchPrefix + "_merged_{0}_XSCALE.LP".format(anomString)
if self.resultsDirectory:
shutil.copy(pathToXSCALELog, os.path.join(self.resultsDirectory, pyarchXSCALELog))
if self.pyarchDirectory is not None:
shutil.copy(pathToXSCALELog, os.path.join(self.pyarchDirectory, pyarchXSCALELog))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXSCALELog
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add XDS_ASCII.HKL if present and gzip it
pathToXdsAsciiHkl = os.path.join(processDirectory, "XDS_ASCII.HKL")
if os.path.exists(pathToXdsAsciiHkl) and self.pyarchDirectory is not None:
pyarchXdsAsciiHkl = self.pyarchPrefix + "_{0}_XDS_ASCII.HKL.gz".format(anomString)
with open(pathToXdsAsciiHkl, 'rb') as f_in:
with gzip.open(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
if self.resultsDirectory:
shutil.copy(os.path.join(self.pyarchDirectory, pyarchXdsAsciiHkl), os.path.join(self.resultsDirectory, pyarchXdsAsciiHkl))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchXdsAsciiHkl
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Result"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add log file
pathToLogFile = edPluginExecAutoPROC.dataOutput.logFile.path.value
autoPROClog = open(pathToLogFile).read()
userString1 = "User : {0} (".format(os.environ["USER"])
userString2 = "User : {0} (".format(proposal)
autoPROClog = autoPROClog.replace(userString1, userString2)
open(pathToLogFile, "w").write(autoPROClog)
pyarchLogFile = self.pyarchPrefix + "_{0}_autoPROC.log".format(anomString)
if self.resultsDirectory:
shutil.copy(pathToLogFile, os.path.join(self.resultsDirectory, pyarchLogFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToLogFile, os.path.join(self.pyarchDirectory, pyarchLogFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchLogFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Add report.pdf
pathToRepordPdf = None
if isStaraniso and edPluginExecAutoPROC.dataOutput.reportPdf_staraniso is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf_staraniso.path.value
elif edPluginExecAutoPROC.dataOutput.reportPdf is not None:
pathToRepordPdf = edPluginExecAutoPROC.dataOutput.reportPdf.path.value
if pathToRepordPdf is not None:
pyarchReportFile = self.pyarchPrefix + "_{0}_{1}".format(anomString, os.path.basename(pathToRepordPdf))
if self.resultsDirectory:
shutil.copy(pathToRepordPdf, os.path.join(self.resultsDirectory, pyarchReportFile))
if self.pyarchDirectory is not None:
shutil.copy(pathToRepordPdf, os.path.join(self.pyarchDirectory, pyarchReportFile))
autoProcProgramAttachment = AutoProcProgramAttachment()
autoProcProgramAttachment.fileName = pyarchReportFile
autoProcProgramAttachment.filePath = self.pyarchDirectory
autoProcProgramAttachment.fileType = "Log"
autoProcProgramContainer.addAutoProcProgramAttachment(autoProcProgramAttachment)
# Upload the xml to ISPyB
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
edPluginStoreAutoprocAnom = self.loadPlugin("EDPluginISPyBStoreAutoProcv1_4", "ISPyBStoreAutoProcv1_4_{0}{1}".format(anomString, staranisoString))
edPluginStoreAutoprocAnom.dataInput = xsDataInputStoreAutoProc
edPluginStoreAutoprocAnom.executeSynchronous()
isSuccess = not edPluginStoreAutoprocAnom.isFailure()
if isSuccess:
self.screen("{0}{1} results uploaded to ISPyB".format(anomString, staranisoString))
if isAnom:
if isStaraniso:
self.hasUploadedAnomStaranisoResultsToISPyB = True
else:
self.hasUploadedAnomResultsToISPyB = True
else:
if isStaraniso:
self.hasUploadedNoanomStaranisoResultsToISPyB = True
else:
self.hasUploadedNoanomResultsToISPyB = True
else:
self.screen("Could not upload {0}{1} results to ISPyB".format(anomString, staranisoString))
from XSDataCommon import XSDataBoolean
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataString
from XSDataCommon import XSDataInteger
from XSDataCommon import XSDataFloat
x = XSDataBoolean(True)
print x.value
print x.marshal()
x = XSDataBoolean(False)
print x.value
print x.marshal()
x = XSDataBoolean('true')
print x.value
print x.marshal()
x = XSDataBoolean('False')
print x.value
print x.marshal()
x = XSDataBoolean(0)
print x.value
print x.marshal()
x = XSDataBoolean(1)
print x.value
print x.marshal()
x = XSDataBoolean(2)
print x.value
print x.marshal()
y = XSDataFloat('this is not a float')
print y.value
print y.marshal()
z = XSDataInteger('this is not an integer')
print z.value
def parseLabelitScreenOutput(self, _strLabelitLogText):
"""
This method parses the labelit.screen log and populates the relevant
parts of the XSDataLabelitScreenOutput object which is then returned.
"""
self.DEBUG("EDPluginLabelitv10.parseLabelitLogText")
xsDataLabelitScreenOutput = None
iIndex = 0
listLogLines = _strLabelitLogText.split('\n')
bFoundLabelitIndexingResults = False
bContinue = True
while (bContinue) :
if (listLogLines[ iIndex ].find("LABELIT Indexing results:") != -1) :
bFoundLabelitIndexingResults = True
bContinue = False
else:
if (iIndex < (len(listLogLines) - 2)):
iIndex += 1
else:
bContinue = False
if (bFoundLabelitIndexingResults == False):
errorMessage = EDMessage.ERROR_EXECUTION_03 % ("EDPluginLabelitv10.parseLabelitLogText", self.getClassName(), "Labelit log message: %s" % _strLabelitLogText)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
else:
# We found some indexing results!
xsDataLabelitScreenOutput = XSDataLabelitScreenOutput()
iIndex += 1
strBeamDistanceMosaicity = listLogLines[ iIndex ]
strBeamDistanceMosaicity = strBeamDistanceMosaicity.replace(',', '')
strBeamDistanceMosaicity = strBeamDistanceMosaicity.replace('mm', ' ')
strBeamDistanceMosaicity = strBeamDistanceMosaicity.replace('=', ' ')
listStringBeamDistanceMosaicity = strBeamDistanceMosaicity.split()
fBeamX = float(listStringBeamDistanceMosaicity[3])
fBeamY = float(listStringBeamDistanceMosaicity[5])
xsDataLabelitScreenOutput.setBeamCentreX(XSDataLength(fBeamX))
xsDataLabelitScreenOutput.setBeamCentreY(XSDataLength(fBeamY))
fDistance = float(listStringBeamDistanceMosaicity[7])
xsDataLabelitScreenOutput.setDistance(XSDataLength(fDistance))
fMosaicity = float(listStringBeamDistanceMosaicity[11])
xsDataLabelitScreenOutput.setMosaicity(XSDataAngle(fMosaicity))
iIndex += 3
bContinue = True
bFoundSelectedSolution = False
while (bContinue):
xsDataLabelitScreenSolution = XSDataLabelitScreenSolution()
strLabelitSolution = listLogLines[ iIndex ]
listStringLabelitSolution = strLabelitSolution.split()
if listStringLabelitSolution[0] == ":)" :
xsDataLabelitScreenSolution.setHappy(XSDataBoolean(True))
else:
xsDataLabelitScreenSolution.setHappy(XSDataBoolean(False))
iSolutionNumber = int(listStringLabelitSolution[1])
xsDataLabelitScreenSolution.setSolutionNumber(XSDataInteger(iSolutionNumber))
fMetricFitValue = float(listStringLabelitSolution[2])
xsDataLabelitScreenSolution.setMetricFitValue(XSDataDouble(fMetricFitValue))
strMetricFitCode = listStringLabelitSolution[3]
xsDataLabelitScreenSolution.setMetricFitCode(XSDataString(strMetricFitCode))
fRMSD = float(listStringLabelitSolution[4])
xsDataLabelitScreenSolution.setRmsd(XSDataLength(fRMSD))
iNumberOfSpots = int(listStringLabelitSolution[5])
xsDataLabelitScreenSolution.setNumberOfSpots(XSDataInteger(iNumberOfSpots))
xsDataLabelitScreenSolution.setCrystalSystem(XSDataString(listStringLabelitSolution[6]))
xsDataLabelitScreenSolution.setBravaisLattice(XSDataString(listStringLabelitSolution[7]))
xsDataCell = XSDataCell()
xsDataCell.setLength_a(XSDataLength(float(listStringLabelitSolution[ 8])))
xsDataCell.setLength_b(XSDataLength(float(listStringLabelitSolution[ 9])))
xsDataCell.setLength_c(XSDataLength(float(listStringLabelitSolution[10])))
xsDataCell.setAngle_alpha(XSDataAngle(float(listStringLabelitSolution[11])))
xsDataCell.setAngle_beta(XSDataAngle(float(listStringLabelitSolution[12])))
xsDataCell.setAngle_gamma(XSDataAngle(float(listStringLabelitSolution[13])))
xsDataLabelitScreenSolution.setUnitCell(xsDataCell)
iVolume = int(listStringLabelitSolution[14])
xsDataLabelitScreenSolution.setVolume(XSDataInteger(iVolume))
# Find the selected solution
if (bFoundSelectedSolution == False):
if (xsDataLabelitScreenSolution.getHappy().getValue() == True):
xsDataLabelitScreenOutput.setSelectedSolutionNumber(XSDataInteger(iSolutionNumber))
bFoundSelectedSolution = True
xsDataLabelitScreenOutput.addLabelitScreenSolution(xsDataLabelitScreenSolution)
if (iIndex < (len(listLogLines) - 2)):
iIndex += 1
else:
bContinue = False
return xsDataLabelitScreenOutput
