def testCopyHTMLFilesAndDir(self):
if not os.path.exists(EDUtilsPath.getEdnaUserTempFolder()):
os.mkdir(EDUtilsPath.getEdnaUserTempFolder())
strTestFromDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(),
"TestFromDir")
shutil.rmtree(strTestFromDir, ignore_errors=True)
os.mkdir(strTestFromDir)
strTestHtmlFilePath = os.path.join(strTestFromDir, "index.html")
strTestHtmlDirPath = os.path.join(strTestFromDir, "index")
EDUtilsFile.writeFile(strTestHtmlFilePath, "Test content")
if not os.path.exists(strTestHtmlDirPath):
os.mkdir(strTestHtmlDirPath)
strTestHtmlDirFilePath = os.path.join(strTestHtmlDirPath, "test.txt")
EDUtilsFile.writeFile(strTestHtmlDirFilePath, "Test content")
#
strTestToDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(),
"TestToDir")
shutil.rmtree(strTestToDir, ignore_errors=True)
os.mkdir(strTestToDir)
EDHandlerESRFPyarchv1_0.copyHTMLDir(strTestFromDir, strTestToDir)
#
# Check that files exist in strTestToDir:
EDAssert.isFile(os.path.join(strTestToDir, "index", "index.html"))
EDAssert.isFile(
os.path.join(strTestToDir, "index", "index", "test.txt"))
#
shutil.rmtree(strTestFromDir, ignore_errors=True)
shutil.rmtree(strTestToDir, ignore_errors=True)
def testCopyHTMLFilesAndDir(self):
if not os.path.exists(EDUtilsPath.getEdnaUserTempFolder()):
os.mkdir(EDUtilsPath.getEdnaUserTempFolder())
strTestFromDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(), "TestFromDir")
shutil.rmtree(strTestFromDir, ignore_errors=True)
os.mkdir(strTestFromDir)
strTestHtmlFilePath = os.path.join(strTestFromDir, "index.html")
strTestHtmlDirPath = os.path.join(strTestFromDir, "index")
EDUtilsFile.writeFile(strTestHtmlFilePath, "Test content")
if not os.path.exists(strTestHtmlDirPath):
os.mkdir(strTestHtmlDirPath)
strTestHtmlDirFilePath = os.path.join(strTestHtmlDirPath, "test.txt")
EDUtilsFile.writeFile(strTestHtmlDirFilePath, "Test content")
#
strTestToDir = os.path.join(EDUtilsPath.getEdnaUserTempFolder(), "TestToDir")
shutil.rmtree(strTestToDir, ignore_errors=True)
os.mkdir(strTestToDir)
EDHandlerESRFPyarchv1_0.copyHTMLDir(strTestFromDir, strTestToDir)
#
# Check that files exist in strTestToDir:
EDAssert.isFile(os.path.join(strTestToDir, "index", "index.html"))
EDAssert.isFile(os.path.join(strTestToDir, "index", "index", "test.txt"))
#
shutil.rmtree(strTestFromDir, ignore_errors=True)
shutil.rmtree(strTestToDir, ignore_errors=True)
def finallyProcess(self, _edPlugin=None):
EDPluginExec.finallyProcess(self, _edPlugin)
self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...")
xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName))))
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory())))
# Write workflowStepReport HTML page
pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName)
pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory())
# Store in Pyarch
if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV() or EDUtilsPath.isALBA:
strPyarchPath = None
if self.xsDataResultCharacterisation is not None:
strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
if strPyarchPath is None:
# For debugging purposes
strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath)))
if not os.path.exists(strPyarchPath):
os.makedirs(strPyarchPath, 0o755)
shutil.copy(pathToJsonFile, strPyarchPath)
pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile))
# Write json file
xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile))
self.setDataOutput(xsDataResultSimpleHTMLPage)
def finallyProcess(self, _edPlugin=None):
EDPluginExec.finallyProcess(self, _edPlugin)
self.DEBUG("EDPluginExecSimpleHTMLPagev1_1.finallyProcess...")
xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), self.strHtmlFileName))))
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(self.getWorkingDirectory())))
# Write workflowStepReport HTML page
pathToIndexFile = self.workflowStepReport.renderHtml(self.getWorkingDirectory(), nameOfIndexFile=self.strHtmlFileName)
pathToJsonFile = self.workflowStepReport.renderJson(self.getWorkingDirectory())
# Store in Pyarch
if EDUtilsPath.isESRF() or EDUtilsPath.isEMBL():
strPyarchPath = None
if self.xsDataResultCharacterisation is not None:
strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
if strPyarchPath is None:
# For debugging purposes
strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(strPyarchPath)))
if not os.path.exists(strPyarchPath):
os.makedirs(strPyarchPath, 0o755)
shutil.copy(pathToJsonFile, strPyarchPath)
pathToJsonFile = os.path.join(strPyarchPath, os.path.basename(pathToJsonFile))
# Write json file
xsDataResultSimpleHTMLPage.pathToJsonFile = XSDataFile(XSDataString(pathToJsonFile))
self.setDataOutput(xsDataResultSimpleHTMLPage)
def finallyProcess(self, _edPlugin=None):
EDPluginExec.finallyProcess(self, _edPlugin)
self.DEBUG("EDPluginExecSimpleHTMLPagev2_0.finallyProcess...")
# Render the page
strHTML = str(self.page)
EDUtilsFile.writeFile(self.strPath, strHTML)
xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(self.strPath)))
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(os.path.dirname(self.strPath))))
self.setDataOutput(xsDataResultSimpleHTMLPage)
# Store in Pyarch
strPyarchPath = None
if self.xsDataResultCharacterisation is not None:
strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
if strPyarchPath is None:
# For debugging purposes
strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
def testCreatePyarchHtmlDirectoryPath(self):
strTestDataDir = self.getPluginTestsDataHome()
strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0", "XSDataCollection_reference.xml")
strXml = self.readAndParseFile(strTestFile)
xsDataCollection = XSDataCollection.parseString(strXml)
strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(xsDataCollection)
# print strPyarchHtmlDirectoryPath
strReferencePath = "/data/pyarch/2016/id30b/opid30b/20161214/RAW_DATA/Magda/Test_x1_2_dnafiles"
EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath, "Correct pyarch path")
def finallyProcess(self, _edPlugin=None):
EDPluginExec.finallyProcess(self, _edPlugin)
self.DEBUG("EDPluginExecSimpleHTMLPagev1_0.finallyProcess...")
# Render the page
strHTML = str(self.page)
EDUtilsFile.writeFile(self.strPath, strHTML)
xsDataResultSimpleHTMLPage = XSDataResultSimpleHTMLPage()
xsDataResultSimpleHTMLPage.setPathToHTMLFile(XSDataFile(XSDataString(self.strPath)))
xsDataResultSimpleHTMLPage.setPathToHTMLDirectory(XSDataFile(XSDataString(os.path.dirname(self.strPath))))
self.setDataOutput(xsDataResultSimpleHTMLPage)
# Store in Pyarch
strPyarchPath = None
if self.xsDataResultCharacterisation is not None:
strPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(self.xsDataResultCharacterisation.getDataCollection())
if strPyarchPath is None:
# For debugging purposes
strPyarchPath = EDUtilsPath.getEdnaUserTempFolder()
EDHandlerESRFPyarchv1_0.copyHTMLDir(_strPathToHTMLDir=os.path.dirname(self.strPath), _strPathToPyarchDirectory=strPyarchPath)
def testCreatePyarchHtmlDirectoryPath(self):
strTestDataDir = self.getPluginTestsDataHome()
strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0", "XSDataCollection_reference.xml")
strXml = self.readAndParseFile(strTestFile)
xsDataCollection = XSDataCollection.parseString(strXml)
strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(xsDataCollection)
#print strPyarchHtmlDirectoryPath
strReferencePath = "/data/pyarch/id23eh1/Pasha/data/visitor/mx1199/id29/20111205/RAW_DATA/4ESR372D11/4ESR372D11_1_dnafiles"
EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath, "Correct pyarch path")
def testCreatePyarchHtmlDirectoryPath(self):
strTestDataDir = self.getPluginTestsDataHome()
strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0", "XSDataCollection_reference.xml")
strXml = self.readAndParseFile(strTestFile)
xsDataCollection = XSDataCollection.parseString(strXml)
strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(xsDataCollection)
#print strPyarchHtmlDirectoryPath
strReferencePath = "/data/pyarch/id23eh1/Pasha/data/visitor/mx1199/id29/20111205/RAW_DATA/4ESR372D11/4ESR372D11_1_dnafiles"
EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath, "Correct pyarch path")
def testCreatePyarchHtmlDirectoryPath(self):
strTestDataDir = self.getPluginTestsDataHome()
strTestFile = os.path.join(strTestDataDir, "EDHandlerESRFPyarchv1_0",
"XSDataCollection_reference.xml")
strXml = self.readAndParseFile(strTestFile)
xsDataCollection = XSDataCollection.parseString(strXml)
strPyarchHtmlDirectoryPath = EDHandlerESRFPyarchv1_0.createPyarchHtmlDirectoryPath(
xsDataCollection)
# print strPyarchHtmlDirectoryPath
strReferencePath = "/data/pyarch/2016/id30b/opid30b/20161214/RAW_DATA/Magda/Test_x1_2_dnafiles"
EDAssert.equal(strReferencePath, strPyarchHtmlDirectoryPath,
"Correct pyarch path")
def testCreatePyarchReprocessDirectoryPath(self):
beamline = "id30a2"
pipelineName = "EDNA_proc"
dataCollectionId = 123456
pyarch_path = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, pipelineName, dataCollectionId)
EDAssert.equal(True, os.path.exists(pyarch_path))
EDAssert.equal(
True,
pyarch_path.startswith(
"/data/pyarch/2020/id30a2/reprocess/EDNA_proc/123456"))
shutil.rmtree(pyarch_path)
def testCreatePyarchReprocessDirectoryPath(self):
beamline = "id30a2"
pipelineName = "EDNA_proc"
dataCollectionId = 123456
pyarch_path = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, pipelineName, dataCollectionId)
EDAssert.equal(True, os.path.exists(pyarch_path))
year = time.strftime("%Y", time.localtime(time.time()))
EDAssert.equal(
True,
pyarch_path.startswith(
"/data/pyarch/{0}/id30a2/reprocess/EDNA_proc/123456".format(
year)))
shutil.rmtree(pyarch_path)
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# Attached files
pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory)
pyarchResultPath = os.path.join(pyarchPath, "results")
if not os.path.exists(pyarchResultPath):
os.makedirs(pyarchResultPath, 0o755)
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0755)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
else:
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
timeStart = time.localtime()
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal, timeStart, timeEnd)
if self.doAnomAndNonanom:
self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal, timeStart, timeEnd)
def testCreatePyarchFilePath(self):
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1/2"))
# Test with inhouse account...
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1/2"))
EDAssert.equal("/data/pyarch/id30a1/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"))
# Test with gzvisitor
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gzvisitor/mx415/id30a3"))
EDAssert.equal("/data/pyarch/id30a3/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212"))
EDAssert.equal("/data/pyarch/id30a3/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212/1"))
EDAssert.equal("/data/pyarch/id30a3/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212/1/2"))
EDAssert.equal("/data/pyarch/id30a3/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525"))
EDAssert.equal("/data/pyarch/id30a3/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525/1"))
EDAssert.equal("/data/pyarch/id30a3/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525/1/2"))
EDAssert.equal("/data/pyarch/id30a3/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"))
def createXSDataInputStoreAutoProc(self,
xsDataResultXDSAPP,
processDirectory,
template,
strPathXscaleLp,
isAnom,
proposal,
timeStart,
timeEnd,
dataCollectionId,
integrationId=None,
programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(
self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[
scalingStatisticsType]:
setattr(
autoProcScalingStatistics, scalingStatisticsAttribute,
dictXscale[scalingStatisticsType]
[scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(
autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId,
status="SUCCESS",
timeStart=timeStart,
timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# Attached files
pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
processDirectory)
pyarchResultPath = os.path.join(pyarchPath, "results")
if not os.path.exists(pyarchResultPath):
os.makedirs(pyarchResultPath, 0o755)
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer,
xsDataResultXDSAPP.logFile.path.value,
"xdsapp",
"log",
isAnom,
attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer,
xsDataResultXDSAPP.pointlessLog.path.value,
"pointless",
"log",
isAnom,
attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer,
xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage",
"log",
isAnom,
attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer,
xsDataResultXDSAPP.correctLP.path.value,
"CORRECT",
"LP",
isAnom,
attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer,
xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII",
"HKL",
isAnom,
attachmentType="Result",
doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer,
xsDataResultXDSAPP.XDS_INP.path.value,
"XDS",
"INP",
isAnom,
attachmentType="Result",
doGzip=False,
noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(
mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer,
mtz_F.path.value,
basenameMtz_F,
"mtz",
isAnom,
attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(
os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer,
mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus,
"mtz",
isAnom,
attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer,
strPathXscaleLp,
"XSCALE",
"LP",
isAnom,
isMerged=True,
attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
strImageFileNameExtension = os.path.splitext(
strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [
".img", ".marccd", ".mccd", ".cbf", ".h5"
]:
self.error(
"Unknown image file name extension for pyarch thumbnail generator: %s"
% strPathToDiffractionImage)
self.setFailure()
else:
# Load the MXWaitFile plugin
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
pathToImageFile = strPathToDiffractionImage
# Quite ugly hack to avoid lag problems at the ESRF:
if EDUtilsPath.isESRF() or EDUtilsPath.isALBA():
if any(beamline in strPathToDiffractionImage
for beamline in ["id23eh1", "id29", "id30b"]):
# Pilatus 6M
self.minImageSize = 6000000
elif any(beamline in strPathToDiffractionImage
for beamline in ["id23eh2", "id30a1"]):
# Pilatus3 2M
self.minImageSize = 2000000
elif strImageFileNameExtension == ".h5":
self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(
pathToImageFile)
pathToImageFile = self.h5DataFilePath
self.isH5 = True
elif EDUtilsPath.isEMBL():
self.minImageSize = 10000
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setFile(
XSDataFile(XSDataString(pathToImageFile)))
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputMXWaitFile.setTimeOut(
self.getDataInput().getWaitForFileTimeOut())
self.edPluginMXWaitFile = self.loadPlugin(
self.strMXWaitFilePluginName)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
# Load the execution plugin
self.edPluginExecThumbnail = self.loadPlugin(
self.strExecThumbnailPluginName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = self.getDataInput(
).getDiffractionImage()
xsDataInputMXThumbnail.height = XSDataInteger(1024)
xsDataInputMXThumbnail.width = XSDataInteger(1024)
xsDataInputMXThumbnail.format = self.dataInput.format
# Output path
strImageNameWithoutExt = os.path.basename(
os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput(
).getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
self.error("Cannot write to forced output directory : %s" %
strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(
strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except Exception as e:
self.WARNING("Couldn't create the directory %s" %
strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
self.warning("Cannot write to pyarch directory: %s" %
strOutputDirname)
strTmpUser = os.path.join("/tmp", os.environ["USER"])
if not os.path.exists(strTmpUser):
os.mkdir(strTmpUser, 0o755)
strOutputDirname = tempfile.mkdtemp(
prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
os.chmod(strOutputDirname, 0o755)
self.warning("Writing thumbnail images to: %s" %
strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(
strOutputDirname, strImageNameWithoutExt)
if self.dataInput.format is not None:
self.strSuffix = self.dataInput.format.value.lower()
self.strImageFormat = self.dataInput.format.value.upper()
self.strOutputPath = os.path.join(
self.strOutputPathWithoutExtension + "." + self.strSuffix)
xsDataInputMXThumbnail.setOutputPath(
XSDataFile(XSDataString(self.strOutputPath)))
self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def testCreatePyarchFilePath(self):
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"),
"/")
EDAssert.equal(None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"),
"/data")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"),
"/data/visitor")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2"), "/data/visitor/mx415/id14eh2")
EDAssert.equal(
"/data/pyarch/2010/id14eh2/mx415/20100212",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2/20100212"),
"/data/visitor/mx415/id14eh2/20100212")
EDAssert.equal(
"/data/pyarch/2010/id14eh2/mx415/20100212/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2/20100212/1"),
"/data/visitor/mx415/id14eh2/20100212/1")
EDAssert.equal(
"/data/pyarch/2010/id14eh2/mx415/20100212/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2/20100212/1/2"),
"/data/visitor/mx415/id14eh2/20100212/1/2")
# Test with inhouse account...
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"),
"/")
EDAssert.equal(None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"),
"/data")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"),
"/data/id23eh2")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse"), "/data/id23eh2/inhouse")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232"),
"/data/id23eh2/inhouse/opid232")
EDAssert.equal(
"/data/pyarch/2010/id23eh2/opid232/20100525",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232/20100525"),
"/data/id23eh2/inhouse/opid232/20100525")
EDAssert.equal(
"/data/pyarch/2010/id23eh2/opid232/20100525/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232/20100525/1"),
"/data/id23eh2/inhouse/opid232/20100525/1")
EDAssert.equal(
"/data/pyarch/2010/id23eh2/opid232/20100525/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232/20100525/1/2"),
"/data/id23eh2/inhouse/opid232/20100525/1/2")
EDAssert.equal(
"/data/pyarch/2014/id30a1/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
),
"/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
)
# Visitor
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3"), "/data/visitor/mx415/id30a3")
EDAssert.equal(
"/data/pyarch/2010/id30a3/mx415/20100212",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3/20100212"),
"/data/visitor/mx415/id30a3/20100212")
EDAssert.equal(
"/data/pyarch/2010/id30a3/mx415/20100212/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3/20100212/1"),
"/data/visitor/mx415/id30a3/20100212/1")
EDAssert.equal(
"/data/pyarch/2010/id30a3/mx415/20100212/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3/20100212/1/2"),
"/data/visitor/mx415/id30a3/20100212/1/2")
EDAssert.equal(
"/data/pyarch/2010/id30a3/opid232/20100525",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid232/20100525"),
"/data/id30a3/inhouse/opid232/20100525")
EDAssert.equal(
"/data/pyarch/2010/id30a3/opid232/20100525/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid232/20100525/1"),
"/data/id30a3/inhouse/opid232/20100525/1")
EDAssert.equal(
"/data/pyarch/2010/id30a3/opid232/20100525/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid232/20100525/1/2"),
"/data/id30a3/inhouse/opid232/20100525/1/2")
EDAssert.equal(
"/data/pyarch/2014/id30a3/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
),
"/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
)
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
EDVerbose.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.getDataInput().getDiffractionImage().getPath().getValue()
strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf"]:
print strImageFileNameExtension
EDVerbose.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage)
self.setFailure()
else:
# Load the waitFile plugin
xsDataInputWaitFile = XSDataInputWaitFile()
xsDataInputWaitFile.setExpectedSize(XSDataInteger(self.__iExpectedSize))
xsDataInputWaitFile.setExpectedFile(self.getDataInput().getDiffractionImage())
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut())
self.__edPluginWaitFile = EDPluginWaitFile()
self.__edPluginWaitFile.setDataInput(xsDataInputWaitFile)
# Load the execution plugin
self.__edPluginExecThumbnail = self.loadPlugin(self.__strExecThumbnailPluginName)
xsDataInputExecThumbnail = XSDataInputExecThumbnail()
xsDataInputExecThumbnail.setInputImagePath(self.getDataInput().getDiffractionImage())
xsDataInputExecThumbnail.setLevelsInvert(XSDataBoolean(True))
xsDataInputExecThumbnail.setLevelsMin(XSDataDoubleWithUnit(0.0))
xsDataDoubleWithUnitLevelsMax = XSDataDoubleWithUnit(99.95)
xsDataDoubleWithUnitLevelsMax.setUnit(XSDataString("%"))
xsDataInputExecThumbnail.setLevelsMax(xsDataDoubleWithUnitLevelsMax)
xsDataInputExecThumbnail.setFilterDilatation([XSDataInteger(4)])
xsDataInputExecThumbnail.setLevelsColorize(XSDataBoolean(False))
xsDataInputExecThumbnail.setThumbHeight(XSDataInteger(1024))
xsDataInputExecThumbnail.setThumbWidth(XSDataInteger(1024))
xsDataInputExecThumbnail.setKeepRatio(XSDataBoolean(False))
# Output path
strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
EDVerbose.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except BaseException, e:
EDVerbose.WARNING("Couldn't create the directory %s" % strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
EDVerbose.warning("Cannot write to pyarch directory: %s" % strOutputDirname)
strOutputDirname = tempfile.mkdtemp("", "EDPluginPyarchThumbnailv10_", "/tmp")
EDVerbose.warning("Writing thumbnail images to: %s" % strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt)
self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + ".jpeg")
xsDataInputExecThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath)))
self.__edPluginExecThumbnail.setDataInput(xsDataInputExecThumbnail)
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if (self.createDNAFileDirectory(strPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPathToDNAFileDirectory)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory)
self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory")
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath))
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin("EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath")
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
elif EDUtilsPath.isMAXIV():
template = ispybDataCollection.fileTemplate
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.fromN is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.fromN.value
if self.dataInput.toN is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.toN.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.path.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
if EDUtilsPath.isEMBL():
fileTemplate = template.replace("#####", "%05d")
else:
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
if EDUtilsPath.isALBA():
_processDirectory = "_".join(pathToStartImage.split('_')[:-1])
from datetime import datetime
_id = datetime.now().strftime('%Y%m%d_%H%M%S')
self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
else:
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"autoPROC", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# The resultsDirectory is not used at ALBA (only pyarchDirectory)
if EDUtilsPath.isALBA():
self.resultsDirectory = None
# Determine pyarch prefix
if EDUtilsPath.isALBA():
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
listPrefix[-2])
else:
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isMAXIV():
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
fWaitFileTimeout = 60 # s
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart)
self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart)
self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart)
# Prepare input to execution plugin
if self.doAnom:
xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCAnom.symm = self.dataInput.symm
xsDataInputAutoPROCAnom.cell = self.dataInput.cell
xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
if self.doNoanom:
xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
xsDataAutoPROCIdentifier.templateN = XSDataString(template)
xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
if self.doAnom:
xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
if self.doNoanom:
xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
if self.doNoanom:
xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
timeStart = time.localtime()
if self.doAnom:
self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
self.edPluginExecAutoPROCAnom.execute()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
self.edPluginExecAutoPROCNoanom.execute()
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
if self.doNoanom:
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
if self.dataInput.startImageNumber is not None:
imageNoStart = self.dataInput.startImageNumber.value
if self.dataInput.endImageNumber is not None:
imageNoEnd = self.dataInput.endImageNumber.value
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if imageNoStart is None:
imageNoStart = ispybDataCollection.startImageNumber
if imageNoEnd is None:
imageNoEnd = ispybDataCollection.startImageNumber + \
ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"XDSAPP", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
if self.dataInput.hdf5ToCbfDirectory is not None:
dir = self.dataInput.hdf5ToCbfDirectory.path.value
pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]
self.timeStart = time.localtime()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
self.edPluginExecXDSAPPAnom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
self.edPluginExecXDSAPPNoanom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXDSAPPAnom.synchronize()
xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
# Run XSCALE even if XSCALE.LP is present
strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
if self.doNoanom:
self.edPluginExecXDSAPPNoanom.synchronize()
xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.dataInput.dataCollectionId is not None:
# Check if we should use XDS_ASCII_to_XML.pl
if self.doAnom:
if self.useXdsAsciiToXml:
# Only for anom runs
self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
else:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlCrystFELv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
userName = os.environ["USER"]
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
#self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
# "CrystFEL",
# "started")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
self.ispyb_retrieve_dc_plugin.executeSynchronous()
ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
#TODO PE2 has 7 digits others 5
template = ispybDataCollection.fileTemplate.replace(
"%07d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
dir_parts = processDirectory.split("/")
processDirectory = "/data/users/%s" % os.path.join(*dir_parts[5:])
# Make results directory
self.resultsDirectory = os.path.join(processDirectory,
"crystfel_results")
try:
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory)
except Exception as ex:
print ex
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
directory)
#TODO: enable when ready
"""
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory)
"""
self.baseFileName = "%s/crystfel_xgandalf_%d" % (
self.resultsDirectory, self.dataInput.dataCollectionId.value)
xsDataInputCrystFEL = XSDataInputCrystFEL()
# Check if imagesFullPath exists. If not then generate one
if self.dataInput.imagesFullPath is None:
imagesFullPath = "%s_images_fullpath.lst" % self.baseName
imagesFullFile = open(imagesFullPath, "w")
for index in range(imageNoEnd - imageNoStart + 1):
imagePath = os.path.join(
directory, ispybDataCollection.fileTemplate % (index + 1))
imagesFullFile.write("%s\n" % imagePath)
imagesFullFile.close()
xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
else:
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
xsDataInputCrystFEL.cellFile = self.dataInput.cellFile
xsDataInputCrystFEL.pointGroup = self.dataInput.pointGroup
xsDataInputCrystFEL.spaceGroup = self.dataInput.spaceGroup
xsDataInputCrystFEL.resCutOff = self.dataInput.resCutOff
xsDataInputCrystFEL.baseFileName = XSDataString(self.baseFileName)
self.timeStart = time.localtime()
self.integration_id, self.program_id = self.create_integration_id(
self.dataInput.dataCollectionId.value, "Creating integration ID")
self.index_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
self.partialator_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataOutput = XSDataResultCrystFEL
self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
self.process_hkl_plugin_odd.process_hkl_type = "o"
self.process_hkl_plugin_even.process_hkl_options += " --even-only"
self.process_hkl_plugin_even.process_hkl_type = "e"
self.index_plugin.executeSynchronous()
if self.index_plugin.isFailure():
self.ERROR('indexamajig: Failed')
self.setFailure()
return
else:
self.screen('indexamajig: Finished')
self.process_hkl_plugin.executeSynchronous()
if self.process_hkl_plugin.isFailure():
self.ERROR('process_hkl: Failed')
self.setFailure()
return
else:
self.screen('process_hkl: Finished')
self.process_hkl_plugin_odd.executeSynchronous()
if self.process_hkl_plugin_odd.isFailure():
self.ERROR('process_hkl_odd: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_odd: Finished')
self.process_hkl_plugin_even.executeSynchronous()
if self.process_hkl_plugin_even.isFailure():
self.ERROR('process_hkl_even: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_even: Finished')
self.partialator_plugin.executeSynchronous()
if self.partialator_plugin.isFailure():
self.ERROR('partialator: Failed')
self.setFailure()
return
else:
self.screen('partialator: Finished')
self.post_process_plugin.executeSynchronous()
if self.post_process_plugin.isFailure():
self.ERROR('post_process: Failed')
self.setFailure()
return
else:
self.screen('post_process: Finished')
self.timeEnd = time.localtime()
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput(
).getResultCharacterisation()
self.xsDataResultMXCuBE.setCharacterisationResult(
xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput(
).getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(
xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(
self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(
strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(
XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(
xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if not EDUtilsPath.isALBA:
if (self.createDNAFileDirectory(strPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation,
strPathToDNAFileDirectory)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
strPathToDNAFileDirectory)
if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation,
strPyArchPathToDNAFileDirectory)
self.xsDataResultMXCuBE.setOutputFileDictionary(
xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage(
).getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary(
).getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption(
).getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML
is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(
XSDataFile(XSDataString(strPathToDNAFileDirectory)),
"dnaFileDirectory")
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult(
)
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult(
)
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar(
) is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar(
).getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(
xsDataResultCharacterisation)
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(
strPyArchPathToDNAFileDirectory,
os.path.basename(strBestWilsonPlotPath))
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(
os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(
os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath,
strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " %
strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath(
)
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(
strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin(
"EDPluginISPyBSetBestWilsonPlotPathv1_4",
"ISPyBSetBestWilsonPlotPath")
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlCrystFELv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
#self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
# "CrystFEL",
# "started")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
self.ispyb_retrieve_dc_plugin.executeSynchronous()
ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
#TODO PE2 has 7 digits others 5
template = ispybDataCollection.fileTemplate.replace("%07d", "####")
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
try:
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 755)
except Exception as ex:
print ex
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
#TODO: enable when ready
"""
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 755)
"""
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 60
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.wait_first_image_plugin.size = XSDataInteger(minSizeFirst)
self.wait_first_image_plugin.dataInput = xsDataInputMXWaitFileFirst
self.wait_first_image_plugin.executeSynchronous()
if self.wait_first_image_plugin.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
self.baseName = "%s/crystfel_xgandalf" % self.resultsDirectory
self.streamFilename = "%s_%d.stream" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
self.hklFilename = "%s_%d.hkl" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
self.mtzFilename = "%s_%d.mtz" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
# Prepare input to execution plugin
xsDataInputCrystFEL = XSDataInputCrystFEL()
# Check if imagesFullPath exists. If not then generate one
if self.dataInput.imagesFullPath is None:
imagesFullPath = "%s_images_fullpath.lst" % self.baseName
imagesFullFile = open(imagesFullPath, "w")
for index in range(imageNoEnd - imageNoStart + 1):
imagePath = os.path.join(directory, ispybDataCollection.fileTemplate % (index + 1))
imagesFullFile.write("%s\n" % imagePath)
imagesFullFile.close()
xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
else:
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
xsDataInputCrystFEL.cellFile = self.dataInput.cellFile
#TODO EDNA can generate the imagesFull path if id do not exist
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.streamFile = XSDataString(self.streamFilename)
xsDataInputCrystFEL.hklFile = XSDataString(self.hklFilename)
xsDataInputCrystFEL.mtzFile = XSDataString(self.mtzFilename)
self.timeStart = time.localtime()
"""
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
self.autoProcIntegrationId, self.autoProcProgramId = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
"""
self.index_plugin.dataInput = xsDataInputCrystFEL
self.index_plugin.executeSynchronous()
if self.index_plugin.isFailure():
self.ERROR('indexamajig: Failed')
self.setFailure()
return
else:
self.screen('indexamajig: Finished')
self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
self.partialator_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
self.process_hkl_plugin_even.process_hkl_options += " --even-only"
self.process_hkl_plugin.executeSynchronous()
if self.process_hkl_plugin.isFailure():
self.ERROR('process_hkl: Failed')
self.setFailure()
return
else:
self.screen('process_hkl: Finished')
self.process_hkl_plugin_odd.dataInput.hklFile = XSDataString(self.hklFilename + "_o")
self.process_hkl_plugin_odd.executeSynchronous()
if self.process_hkl_plugin_odd.isFailure():
self.ERROR('process_hkl_odd: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_odd: Finished')
self.process_hkl_plugin_even.dataInput.hklFile = XSDataString(self.hklFilename + "_e")
self.process_hkl_plugin_even.executeSynchronous()
if self.process_hkl_plugin_even.isFailure():
self.ERROR('process_hkl_even: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_even: Finished')
self.partialator_plugin.executeSynchronous()
if self.partialator_plugin.isFailure():
self.ERROR('partialator: Failed')
self.setFailure()
return
else:
self.screen('partialator: Finished')
self.post_process_plugin.executeSynchronous()
if self.post_process_plugin.isFailure():
self.ERROR('post_process: Failed')
self.setFailure()
return
else:
self.screen('post_process: Finished')
"""
self.edPluginExecCrystFELScale.dataInput = xsDataInputCrystFEL
self.edPluginExecCrystFELScale.executeSynchronous()
if self.edPluginExecCrystFELScale.isFailure():
self.ERROR('CrystFELScalev1_0: Failed')
self.setFailure()
return
else:
self.screen('CrystFELScalev1_0: Finished')
self.timeEnd = time.localtime()
"""
# Upload to ISPyB
"""
def testCreatePyarchFilePath(self):
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"))
EDAssert.equal("/data/pyarch/id14eh2/mx415", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1/2"))
# Test with inhouse account...
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse"))
EDAssert.equal("/data/pyarch/id23eh2/opid232", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1/2"))
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
if self.bIsEigerDetector:
xsDataResultCharacterisation = self.makeNumberOfImagesMultipleOf100(xsDataResultCharacterisation)
self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if self.createDNAFileDirectory(strPathToDNAFileDirectory):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation, strPathToDNAFileDirectory
)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
if self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory
)
self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan is not None and xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(
XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory"
)
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
strBestWilsonPlotPyarchPath = None
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(
strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath)
)
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = (
self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
)
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin(
"EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath"
)
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
# Only for the ESRF:
if EDUtilsPath.isESRF():
# For EXI: create workflow entry with one workflow step
self.edPluginStoreWorkflow = self.loadPlugin(self.strPluginStoreWorkflow)
self.edPluginStoreWorkflowStep = self.loadPlugin(self.strPluginStoreWorkflowStep)
self.edPluginUpdateDataCollectionGroupWorkflowId = self.loadPlugin(
self.strUpdateDataCollectionGroupWorkflowId
)
xsDataISPyBWorkflow = XSDataISPyBWorkflow()
xsDataISPyBWorkflow.workflowType = XSDataString("Characterisation")
xsDataISPyBWorkflow.workflowTitle = XSDataString("Characterisation")
xsDataInputISPyBStoreWorkflow = XSDataInputISPyBStoreWorkflow()
xsDataInputISPyBStoreWorkflow.workflow = xsDataISPyBWorkflow
self.edPluginStoreWorkflow.dataInput = xsDataInputISPyBStoreWorkflow
self.edPluginStoreWorkflow.executeSynchronous()
if (
self.edPluginStoreWorkflow.dataOutput is not None
and self.edPluginStoreWorkflow.dataOutput.workflowId is not None
):
workflowId = self.edPluginStoreWorkflow.dataOutput.workflowId.value
# Update data collection group
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId = (
XSDataInputISPyBUpdateDataCollectionGroupWorkflowId()
)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.workflowId = XSDataInteger(workflowId)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileLocation = XSDataString(
os.path.dirname(self.xsDataFirstImage.path.value)
)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileName = XSDataString(
os.path.basename(self.xsDataFirstImage.path.value)
)
self.edPluginUpdateDataCollectionGroupWorkflowId.dataInput = (
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId
)
self.edPluginUpdateDataCollectionGroupWorkflowId.executeSynchronous()
xsDataInputISPyBStoreWorkflowStep = XSDataInputISPyBStoreWorkflowStep()
xsDataInputISPyBStoreWorkflowStep.workflowId = XSDataInteger(workflowId)
xsDataInputISPyBStoreWorkflowStep.workflowStepType = XSDataString("Characterisation")
xsDataInputISPyBStoreWorkflowStep.status = XSDataString("Success")
if strBestWilsonPlotPyarchPath is not None:
xsDataInputISPyBStoreWorkflowStep.imageResultFilePath = XSDataString(
strBestWilsonPlotPyarchPath
)
xsDataInputISPyBStoreWorkflowStep.htmlResultFilePath = XSDataString(strPyArchPathToDNAFileDirectory)
if self.edPluginExecSimpleHTML.dataOutput is not None:
strResultFilePath = self.edPluginExecSimpleHTML.dataOutput.pathToJsonFile.path.value
# strPyarchResultFilePath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strResultFilePath)
xsDataInputISPyBStoreWorkflowStep.resultFilePath = XSDataString(strResultFilePath)
self.edPluginStoreWorkflowStep.dataInput = xsDataInputISPyBStoreWorkflowStep
self.edPluginStoreWorkflowStep.executeSynchronous()
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlDozorv1_0.postProcess")
# Write a file to be used with ISPyB or GNUPLOT only if data collection id in input
dataCollectionId = None
if self.dataInput.dataCollectionId is None and len(self.dataInput.image) > 0:
# Only try to obtain data collection id if at the ESRF and path starts with "/data"
if EDUtilsPath.isESRF() and self.dataInput.image[0].path.value.startswith("/data"):
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.image = XSDataImage(self.dataInput.image[0].path)
edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginISPyBRetrieveDataCollection.executeSynchronous()
xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput
if xsDataResultRetrieveDataCollection is not None:
dataCollection = xsDataResultRetrieveDataCollection.dataCollection
if dataCollection is not None:
dataCollectionId = dataCollection.dataCollectionId
elif self.dataInput.dataCollectionId is not None:
dataCollectionId = self.dataInput.dataCollectionId.value
if dataCollectionId is not None:
minImageNumber = None
maxImageNumber = None
minAngle = None
maxAngle = None
minDozorValue = None
maxDozorValue = None
minResolution = None
maxResolution = None
dozorPlotFileName = "dozor_{0}.png".format(dataCollectionId)
dozorCsvFileName = "dozor_{0}.csv".format(dataCollectionId)
with open(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), "w") as gnuplotFile:
gnuplotFile.write("# Data directory: {0}\n".format(self.directory))
gnuplotFile.write("# File template: {0}\n".format(self.template.replace("%04d", "####")))
gnuplotFile.write("# {0:>9s}{1:>16s}{2:>16s}{3:>16s}{4:>16s}{5:>16s}\n".format("'Image no'",
"'Angle'",
"'No of spots'",
"'Main score (*10)'",
"'Spot score'",
"'Visible res.'",
))
for imageDozor in self.dataOutput.imageDozor:
gnuplotFile.write("{0:10d},{1:15.3f},{2:15d},{3:15.3f},{4:15.3f},{5:15.3f}\n".format(imageDozor.number.value,
imageDozor.angle.value,
imageDozor.spotsNumOf.value,
10 * imageDozor.mainScore.value,
imageDozor.spotScore.value,
imageDozor.visibleResolution.value,
))
if minImageNumber is None or minImageNumber > imageDozor.number.value:
minImageNumber = imageDozor.number.value
minAngle = imageDozor.angle.value
if maxImageNumber is None or maxImageNumber < imageDozor.number.value:
maxImageNumber = imageDozor.number.value
maxAngle = imageDozor.angle.value
if minDozorValue is None or minDozorValue > imageDozor.mainScore.value:
minDozorValue = imageDozor.spotScore.value
if maxDozorValue is None or maxDozorValue < imageDozor.mainScore.value:
maxDozorValue = imageDozor.spotScore.value
# Min resolution: the higher the value the lower the resolution
if minResolution is None or minResolution < imageDozor.visibleResolution.value:
# Disregard resolution worse than 10.0
if imageDozor.visibleResolution.value < 10.0:
minResolution = imageDozor.visibleResolution.value
# Max resolution: the lower the number the better the resolution
if maxResolution is None or maxResolution > imageDozor.visibleResolution.value:
maxResolution = imageDozor.visibleResolution.value
xtics = ""
if minImageNumber is not None and minImageNumber == maxImageNumber:
minAngle -= 1.0
maxAngle += 1.0
noImages = maxImageNumber - minImageNumber + 1
if noImages <= 4:
minImageNumber -= 0.1
maxImageNumber += 0.1
deltaAngle = maxAngle - minAngle
minAngle -= deltaAngle * 0.1 / noImages
maxAngle += deltaAngle * 0.1 / noImages
xtics = "1"
if maxResolution is None or maxResolution > 0.8:
maxResolution = 0.8
else:
maxResolution = int(maxResolution * 10.0) / 10.0
if minResolution is None or minResolution < 4.5:
minResolution = 4.5
else:
minResolution = int(minResolution * 10.0) / 10.0 + 1
if maxDozorValue < 0.001 and minDozorValue < 0.001:
yscale = "set yrange [-0.5:0.5]\n set ytics 1"
else:
yscale = "set autoscale y"
gnuplotFile.close()
gnuplotScript = \
"""#
set terminal png
set output '{dozorPlotFileName}'
set title '{title}'
set grid x2 y2
set xlabel 'Image number'
set x2label 'Angle (degrees)'
set y2label 'Resolution (A)'
set ylabel 'Number of spots / Dozor score (*10)'
set xtics {xtics} nomirror
set x2tics
set ytics nomirror
set y2tics
set xrange [{minImageNumber}:{maxImageNumber}]
set x2range [{minAngle}:{maxAngle}]
{yscale}
set y2range [{minResolution}:{maxResolution}]
set key below
plot '{dozorCsvFileName}' using 1:3 title 'Number of spots' axes x1y1 with points linetype rgb 'goldenrod' pointtype 7 pointsize 1.5, \
'{dozorCsvFileName}' using 1:4 title 'Dozor score' axes x1y1 with points linetype 3 pointtype 7 pointsize 1.5, \
'{dozorCsvFileName}' using 1:6 title 'Visible resolution' axes x1y2 with points linetype 1 pointtype 7 pointsize 1.5
""".format(title=self.template.replace("%04d", "####"),
dozorPlotFileName=dozorPlotFileName,
dozorCsvFileName=dozorCsvFileName,
minImageNumber=minImageNumber,
maxImageNumber=maxImageNumber,
minAngle=minAngle,
maxAngle=maxAngle,
minResolution=minResolution,
maxResolution=maxResolution,
xtics=xtics,
yscale=yscale,
)
pathGnuplotScript = os.path.join(self.getWorkingDirectory(), "gnuplot.sh")
data_file = open(pathGnuplotScript, "w")
data_file.write(gnuplotScript)
data_file.close()
oldCwd = os.getcwd()
os.chdir(self.getWorkingDirectory())
os.system("{0} {1}".format(self.gnuplot, pathGnuplotScript))
os.chdir(oldCwd)
self.dataOutput.dozorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), dozorPlotFileName)))
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = os.path.join(self.directory.replace("RAW_DATA", "PROCESSED_DATA"), "DozorPlot")
resultsDirectory = os.path.join(processDirectory, "results")
dozorPlotResultPath = os.path.join(resultsDirectory, dozorPlotFileName)
dozorCsvResultPath = os.path.join(resultsDirectory, dozorCsvFileName)
try:
if not os.path.exists(resultsDirectory):
os.makedirs(resultsDirectory, 0o755)
shutil.copy(os.path.join(self.getWorkingDirectory(), dozorPlotFileName), dozorPlotResultPath)
shutil.copy(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), dozorCsvResultPath)
except:
self.warning("Couldn't copy files to results directory: {0}".format(resultsDirectory))
try:
# Create paths on pyarch
dozorPlotPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorPlotResultPath)
dozorCsvPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorCsvResultPath)
if not os.path.exists(os.path.dirname(dozorPlotPyarchPath)):
os.makedirs(os.path.dirname(dozorPlotPyarchPath), 0o755)
shutil.copy(dozorPlotResultPath, dozorPlotPyarchPath)
shutil.copy(dozorCsvResultPath, dozorCsvPyarchPath)
# Upload to data collection
xsDataInputISPyBSetImageQualityIndicatorsPlot = XSDataInputISPyBSetImageQualityIndicatorsPlot()
xsDataInputISPyBSetImageQualityIndicatorsPlot.dataCollectionId = XSDataInteger(dataCollectionId)
xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsPlotPath = XSDataString(dozorPlotPyarchPath)
xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsCSVPath = XSDataString(dozorCsvPyarchPath)
EDPluginISPyBSetImageQualityIndicatorsPlot = self.loadPlugin("EDPluginISPyBSetImageQualityIndicatorsPlotv1_4")
EDPluginISPyBSetImageQualityIndicatorsPlot.dataInput = xsDataInputISPyBSetImageQualityIndicatorsPlot
EDPluginISPyBSetImageQualityIndicatorsPlot.executeSynchronous()
except:
self.warning("Couldn't copy files to pyarch: {0}".format(dozorPlotPyarchPath))
self.sendMessageToMXCuBE("Processing finished", "info")
self.setStatusToMXCuBE("Success")
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf", ".h5"]:
self.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage)
self.setFailure()
else:
# Load the MXWaitFile plugin
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
pathToImageFile = strPathToDiffractionImage
# Quite ugly hack to avoid lag problems at the ESRF:
if EDUtilsPath.isESRF():
if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]):
# Pilatus 6M
self.minImageSize = 6000000
elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]):
# Pilatus3 2M
self.minImageSize = 2000000
elif strImageFileNameExtension == ".h5":
self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(pathToImageFile)
pathToImageFile = self.h5DataFilePath
self.isH5 = True
elif EDUtilsPath.isEMBL():
self.minImageSize = 10000
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(pathToImageFile)))
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputMXWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut())
self.edPluginMXWaitFile = self.loadPlugin(self.strMXWaitFilePluginName)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
# Load the execution plugin
self.edPluginExecThumbnail = self.loadPlugin(self.strExecThumbnailPluginName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = self.getDataInput().getDiffractionImage()
xsDataInputMXThumbnail.height = XSDataInteger(1024)
xsDataInputMXThumbnail.width = XSDataInteger(1024)
xsDataInputMXThumbnail.format = self.dataInput.format
# Output path
strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except Exception as e:
self.WARNING("Couldn't create the directory %s" % strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
self.warning("Cannot write to pyarch directory: %s" % strOutputDirname)
strTmpUser = os.path.join("/tmp", os.environ["USER"])
if not os.path.exists(strTmpUser):
os.mkdir(strTmpUser, 0755)
strOutputDirname = tempfile.mkdtemp(prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
os.chmod(strOutputDirname, 0755)
self.warning("Writing thumbnail images to: %s" % strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt)
if self.dataInput.format is not None:
self.strSuffix = self.dataInput.format.value.lower()
self.strImageFormat = self.dataInput.format.value.upper()
self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + "." + self.strSuffix)
xsDataInputMXThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath)))
self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doNoanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(masterFilePath)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(masterFilePath)))
else:
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
if self.doAnom:
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
