def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.dataInput.diffractionImage.path.value
strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf", ".h5"]:
self.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage)
self.setFailure()
else:
# Load the MXWaitFile plugin
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
pathToImageFile = strPathToDiffractionImage
# Quite ugly hack to avoid lag problems at the ESRF:
if EDUtilsPath.isESRF():
if any(beamline in strPathToDiffractionImage for beamline in ["id23eh1", "id29", "id30b"]):
# Pilatus 6M
self.minImageSize = 6000000
elif any(beamline in strPathToDiffractionImage for beamline in ["id23eh2", "id30a1"]):
# Pilatus3 2M
self.minImageSize = 2000000
elif strImageFileNameExtension == ".h5":
self.h5MasterFilePath, self.h5DataFilePath, self.h5FileNumber = self.getH5FilePath(pathToImageFile)
pathToImageFile = self.h5DataFilePath
self.isH5 = True
elif EDUtilsPath.isEMBL():
self.minImageSize = 10000
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setFile(XSDataFile(XSDataString(pathToImageFile)))
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputMXWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut())
self.edPluginMXWaitFile = self.loadPlugin(self.strMXWaitFilePluginName)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
# Load the execution plugin
self.edPluginExecThumbnail = self.loadPlugin(self.strExecThumbnailPluginName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = self.getDataInput().getDiffractionImage()
xsDataInputMXThumbnail.height = XSDataInteger(1024)
xsDataInputMXThumbnail.width = XSDataInteger(1024)
xsDataInputMXThumbnail.format = self.dataInput.format
# Output path
strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
self.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except Exception as e:
self.WARNING("Couldn't create the directory %s" % strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
self.warning("Cannot write to pyarch directory: %s" % strOutputDirname)
strTmpUser = os.path.join("/tmp", os.environ["USER"])
if not os.path.exists(strTmpUser):
os.mkdir(strTmpUser, 0755)
strOutputDirname = tempfile.mkdtemp(prefix="EDPluginPyarchThumbnailv10_", dir=strTmpUser)
os.chmod(strOutputDirname, 0755)
self.warning("Writing thumbnail images to: %s" % strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt)
if self.dataInput.format is not None:
self.strSuffix = self.dataInput.format.value.lower()
self.strImageFormat = self.dataInput.format.value.upper()
self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + "." + self.strSuffix)
xsDataInputMXThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath)))
self.edPluginExecThumbnail.setDataInput(xsDataInputMXThumbnail)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0755)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(masterFilePath)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(masterFilePath)))
else:
xsDataInputXia2DIALSAnom.addImage(XSDataFile(XSDataString(pathToStartImage)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(XSDataFile(XSDataString(pathToStartImage)))
timeStart = time.localtime()
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
self.uploadToISPyB(self.edPluginExecXia2DIALSAnom, True, proposal, timeStart, timeEnd)
if self.doAnomAndNonanom:
self.uploadToISPyB(self.edPluginExecXia2DIALSNoanom, False, proposal, timeStart, timeEnd)
def testCreatePyarchFilePath(self):
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1/2"))
# Test with inhouse account...
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1/2"))
EDAssert.equal("/data/pyarch/id30a1/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"))
# Test with gzvisitor
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gzvisitor/mx415/id30a3"))
EDAssert.equal("/data/pyarch/id30a3/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212"))
EDAssert.equal("/data/pyarch/id30a3/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212/1"))
EDAssert.equal("/data/pyarch/id30a3/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/gz/visitor/mx415/id30a3/20100212/1/2"))
EDAssert.equal("/data/pyarch/id30a3/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525"))
EDAssert.equal("/data/pyarch/id30a3/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525/1"))
EDAssert.equal("/data/pyarch/id30a3/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid232/20100525/1/2"))
EDAssert.equal("/data/pyarch/id30a3/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"))
def createXSDataInputStoreAutoProc(self, xsDataResultXDSAPP, processDirectory, template,
strPathXscaleLp, isAnom, proposal, timeStart, timeEnd, dataCollectionId,
integrationId=None, programId=None):
# Parse log file
dictLog = self.parseLogFile(xsDataResultXDSAPP.logFile.path.value)
dictXscale = self.parseXscaleLp(strPathXscaleLp)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
xsDataInputStoreAutoProc = XSDataInputStoreAutoProc()
autoProcContainer = AutoProcContainer()
# AutoProc
autoProc = AutoProc()
autoProc.spaceGroup = dictLog["spaceGroup"]
autoProc.refinedCell_a = dictLog["cellA"]
autoProc.refinedCell_b = dictLog["cellB"]
autoProc.refinedCell_c = dictLog["cellC"]
autoProc.refinedCell_alpha = dictLog["cellAlpha"]
autoProc.refinedCell_beta = dictLog["cellBeta"]
autoProc.refinedCell_gamma = dictLog["cellGamma"]
autoProcContainer.AutoProc = autoProc
# AutoProcIntegrationContainer
autoProcIntegrationContainer = AutoProcIntegrationContainer()
autoProcIntegration = AutoProcIntegration()
autoProcIntegration.autoProcIntegrationId = integrationId
autoProcIntegration.cell_a = dictLog["cellA"]
autoProcIntegration.cell_b = dictLog["cellB"]
autoProcIntegration.cell_c = dictLog["cellC"]
autoProcIntegration.cell_alpha = dictLog["cellAlpha"]
autoProcIntegration.cell_beta = dictLog["cellBeta"]
autoProcIntegration.cell_gamma = dictLog["cellGamma"]
autoProcIntegration.anomalous = isAnom
image = Image()
image.dataCollectionId = dataCollectionId
autoProcIntegrationContainer.AutoProcIntegration = autoProcIntegration
autoProcIntegrationContainer.Image = image
# Scaling container
if xsDataResultXDSAPP.correctLP is not None:
isa = self.parseCorrectLp(xsDataResultXDSAPP.correctLP.path.value)
else:
isa = None
autoProcScalingContainer = AutoProcScalingContainer()
autoProcScaling = AutoProcScaling()
autoProcScaling.recordTimeStamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
autoProcScalingContainer.AutoProcScaling = autoProcScaling
for scalingStatisticsType in dictXscale:
autoProcScalingStatistics = AutoProcScalingStatistics()
autoProcScalingStatistics.scalingStatisticsType = scalingStatisticsType
autoProcScalingStatistics.anomalous = isAnom
for scalingStatisticsAttribute in dictXscale[scalingStatisticsType]:
setattr(autoProcScalingStatistics, scalingStatisticsAttribute, dictXscale[scalingStatisticsType][scalingStatisticsAttribute])
if scalingStatisticsType == "overall" and isa is not None:
autoProcScalingStatistics.isa = isa
autoProcScalingContainer.addAutoProcScalingStatistics(autoProcScalingStatistics)
autoProcScalingContainer.AutoProcIntegrationContainer = autoProcIntegrationContainer
autoProcContainer.AutoProcScalingContainer = autoProcScalingContainer
# Program
autoProcProgramContainer = AutoProcProgramContainer()
autoProcProgram = EDHandlerXSDataISPyBv1_4.createAutoProcProgram(
programId=programId, status="SUCCESS", timeStart=timeStart, timeEnd=timeEnd,
processingCommandLine=self.processingCommandLine, processingPrograms=self.processingPrograms)
autoProcProgramContainer.AutoProcProgram = autoProcProgram
# Attached files
pyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(processDirectory)
pyarchResultPath = os.path.join(pyarchPath, "results")
if not os.path.exists(pyarchResultPath):
os.makedirs(pyarchResultPath, 0o755)
# XDSAPP log and result files
if xsDataResultXDSAPP.logFile is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.logFile.path.value,
"xdsapp", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.pointlessLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.pointlessLog.path.value,
"pointless", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.phenixXtriageLog is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.phenixXtriageLog.path.value,
"xtriage", "log", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.correctLP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.correctLP.path.value,
"CORRECT", "LP", isAnom, attachmentType="Log")
if xsDataResultXDSAPP.XDS_ASCII_HKL is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_ASCII_HKL.path.value,
"XDS_ASCII", "HKL", isAnom, attachmentType="Result", doGzip=True)
if xsDataResultXDSAPP.XDS_INP is not None:
self.addAttachment(autoProcProgramContainer, xsDataResultXDSAPP.XDS_INP.path.value,
"XDS", "INP", isAnom, attachmentType="Result", doGzip=False, noMergedString=True)
for mtz_F in xsDataResultXDSAPP.mtz_F:
basenameMtz_F = os.path.splitext(os.path.basename(mtz_F.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F.path.value,
basenameMtz_F, "mtz", isAnom, attachmentType="Result")
for mtz_F_plus_F_minus in xsDataResultXDSAPP.mtz_F_plus_F_minus:
basenameMtz_F_plus_F_minus = os.path.splitext(os.path.basename(mtz_F_plus_F_minus.path.value))[0]
self.addAttachment(autoProcProgramContainer, mtz_F_plus_F_minus.path.value,
basenameMtz_F_plus_F_minus, "mtz", isAnom, attachmentType="Result")
# for mtz_I in xsDataResultXDSAPP.mtz_I:
# basenameMtz_I = os.path.splitext(os.path.basename(mtz_I.path.value))[0]
# self.addAttachment(autoProcProgramContainer, mtz_I.path.value,
# basenameMtz_I, "mtz", isAnom, attachmentType="Result")
# for hkl in xsDataResultXDSAPP.hkl:
# basenameHkl = os.path.splitext(os.path.basename(hkl.path.value))[0]
# self.addAttachment(autoProcProgramContainer, hkl.path.value,
# basenameHkl, "hkl", isAnom, attachmentType="Result", doGzip=True)
# for cv in xsDataResultXDSAPP.cv:
# basenameCv = os.path.splitext(os.path.basename(cv.path.value))[0]
# self.addAttachment(autoProcProgramContainer, cv.path.value,
# basenameCv, "cv", isAnom, attachmentType="Result", doGzip=True)
if os.path.exists(strPathXscaleLp):
self.addAttachment(autoProcProgramContainer, strPathXscaleLp,
"XSCALE", "LP", isAnom, isMerged=True, attachmentType="Result")
autoProcContainer.AutoProcProgramContainer = autoProcProgramContainer
xsDataInputStoreAutoProc.AutoProcContainer = autoProcContainer
return xsDataInputStoreAutoProc
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
EDVerbose.DEBUG("EDPluginControlPyarchThumbnailGeneratorv1_0.preProcess")
# Check that the input image exists and is of the expected type
strPathToDiffractionImage = self.getDataInput().getDiffractionImage().getPath().getValue()
strImageFileNameExtension = os.path.splitext(strPathToDiffractionImage)[1]
if not strImageFileNameExtension in [".img", ".marccd", ".mccd", ".cbf"]:
print strImageFileNameExtension
EDVerbose.error("Unknown image file name extension for pyarch thumbnail generator: %s" % strPathToDiffractionImage)
self.setFailure()
else:
# Load the waitFile plugin
xsDataInputWaitFile = XSDataInputWaitFile()
xsDataInputWaitFile.setExpectedSize(XSDataInteger(self.__iExpectedSize))
xsDataInputWaitFile.setExpectedFile(self.getDataInput().getDiffractionImage())
if self.getDataInput().getWaitForFileTimeOut():
xsDataInputWaitFile.setTimeOut(self.getDataInput().getWaitForFileTimeOut())
self.__edPluginWaitFile = EDPluginWaitFile()
self.__edPluginWaitFile.setDataInput(xsDataInputWaitFile)
# Load the execution plugin
self.__edPluginExecThumbnail = self.loadPlugin(self.__strExecThumbnailPluginName)
xsDataInputExecThumbnail = XSDataInputExecThumbnail()
xsDataInputExecThumbnail.setInputImagePath(self.getDataInput().getDiffractionImage())
xsDataInputExecThumbnail.setLevelsInvert(XSDataBoolean(True))
xsDataInputExecThumbnail.setLevelsMin(XSDataDoubleWithUnit(0.0))
xsDataDoubleWithUnitLevelsMax = XSDataDoubleWithUnit(99.95)
xsDataDoubleWithUnitLevelsMax.setUnit(XSDataString("%"))
xsDataInputExecThumbnail.setLevelsMax(xsDataDoubleWithUnitLevelsMax)
xsDataInputExecThumbnail.setFilterDilatation([XSDataInteger(4)])
xsDataInputExecThumbnail.setLevelsColorize(XSDataBoolean(False))
xsDataInputExecThumbnail.setThumbHeight(XSDataInteger(1024))
xsDataInputExecThumbnail.setThumbWidth(XSDataInteger(1024))
xsDataInputExecThumbnail.setKeepRatio(XSDataBoolean(False))
# Output path
strImageNameWithoutExt = os.path.basename(os.path.splitext(strPathToDiffractionImage)[0])
strImageDirname = os.path.dirname(strPathToDiffractionImage)
if self.getDataInput().getForcedOutputDirectory():
strForcedOutputDirectory = self.getDataInput().getForcedOutputDirectory().getPath().getValue()
if not os.access(strForcedOutputDirectory, os.W_OK):
EDVerbose.error("Cannot write to forced output directory : %s" % strForcedOutputDirectory)
self.setFailure()
else:
self.strOutputPathWithoutExtension = os.path.join(strForcedOutputDirectory, strImageNameWithoutExt)
else:
# Try to store in the ESRF pyarch directory
strOutputDirname = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strImageDirname)
# Check that output pyarch path exists and is writeable:
bIsOk = False
if strOutputDirname:
if not os.path.exists(strOutputDirname):
# Try to create the directory
try:
os.makedirs(strOutputDirname)
bIsOk = True
except BaseException, e:
EDVerbose.WARNING("Couldn't create the directory %s" % strOutputDirname)
elif os.access(strOutputDirname, os.W_OK):
bIsOk = True
if not bIsOk:
EDVerbose.warning("Cannot write to pyarch directory: %s" % strOutputDirname)
strOutputDirname = tempfile.mkdtemp("", "EDPluginPyarchThumbnailv10_", "/tmp")
EDVerbose.warning("Writing thumbnail images to: %s" % strOutputDirname)
self.strOutputPathWithoutExtension = os.path.join(strOutputDirname, strImageNameWithoutExt)
self.strOutputPath = os.path.join(self.strOutputPathWithoutExtension + ".jpeg")
xsDataInputExecThumbnail.setOutputPath(XSDataFile(XSDataString(self.strOutputPath)))
self.__edPluginExecThumbnail.setDataInput(xsDataInputExecThumbnail)
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlDozorv1_0.postProcess")
# Write a file to be used with ISPyB or GNUPLOT only if data collection id in input
dataCollectionId = None
if self.dataInput.dataCollectionId is None and len(self.dataInput.image) > 0:
# Only try to obtain data collection id if at the ESRF and path starts with "/data"
if EDUtilsPath.isESRF() and self.dataInput.image[0].path.value.startswith("/data"):
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.image = XSDataImage(self.dataInput.image[0].path)
edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginISPyBRetrieveDataCollection.executeSynchronous()
xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput
if xsDataResultRetrieveDataCollection is not None:
dataCollection = xsDataResultRetrieveDataCollection.dataCollection
if dataCollection is not None:
dataCollectionId = dataCollection.dataCollectionId
elif self.dataInput.dataCollectionId is not None:
dataCollectionId = self.dataInput.dataCollectionId.value
if dataCollectionId is not None:
minImageNumber = None
maxImageNumber = None
minAngle = None
maxAngle = None
minDozorValue = None
maxDozorValue = None
minResolution = None
maxResolution = None
dozorPlotFileName = "dozor_{0}.png".format(dataCollectionId)
dozorCsvFileName = "dozor_{0}.csv".format(dataCollectionId)
with open(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), "w") as gnuplotFile:
gnuplotFile.write("# Data directory: {0}\n".format(self.directory))
gnuplotFile.write("# File template: {0}\n".format(self.template.replace("%04d", "####")))
gnuplotFile.write("# {0:>9s}{1:>16s}{2:>16s}{3:>16s}{4:>16s}{5:>16s}\n".format("'Image no'",
"'Angle'",
"'No of spots'",
"'Main score (*10)'",
"'Spot score'",
"'Visible res.'",
))
for imageDozor in self.dataOutput.imageDozor:
gnuplotFile.write("{0:10d},{1:15.3f},{2:15d},{3:15.3f},{4:15.3f},{5:15.3f}\n".format(imageDozor.number.value,
imageDozor.angle.value,
imageDozor.spotsNumOf.value,
10 * imageDozor.mainScore.value,
imageDozor.spotScore.value,
imageDozor.visibleResolution.value,
))
if minImageNumber is None or minImageNumber > imageDozor.number.value:
minImageNumber = imageDozor.number.value
minAngle = imageDozor.angle.value
if maxImageNumber is None or maxImageNumber < imageDozor.number.value:
maxImageNumber = imageDozor.number.value
maxAngle = imageDozor.angle.value
if minDozorValue is None or minDozorValue > imageDozor.mainScore.value:
minDozorValue = imageDozor.spotScore.value
if maxDozorValue is None or maxDozorValue < imageDozor.mainScore.value:
maxDozorValue = imageDozor.spotScore.value
# Min resolution: the higher the value the lower the resolution
if minResolution is None or minResolution < imageDozor.visibleResolution.value:
# Disregard resolution worse than 10.0
if imageDozor.visibleResolution.value < 10.0:
minResolution = imageDozor.visibleResolution.value
# Max resolution: the lower the number the better the resolution
if maxResolution is None or maxResolution > imageDozor.visibleResolution.value:
maxResolution = imageDozor.visibleResolution.value
xtics = ""
if minImageNumber is not None and minImageNumber == maxImageNumber:
minAngle -= 1.0
maxAngle += 1.0
noImages = maxImageNumber - minImageNumber + 1
if noImages <= 4:
minImageNumber -= 0.1
maxImageNumber += 0.1
deltaAngle = maxAngle - minAngle
minAngle -= deltaAngle * 0.1 / noImages
maxAngle += deltaAngle * 0.1 / noImages
xtics = "1"
if maxResolution is None or maxResolution > 0.8:
maxResolution = 0.8
else:
maxResolution = int(maxResolution * 10.0) / 10.0
if minResolution is None or minResolution < 4.5:
minResolution = 4.5
else:
minResolution = int(minResolution * 10.0) / 10.0 + 1
if maxDozorValue < 0.001 and minDozorValue < 0.001:
yscale = "set yrange [-0.5:0.5]\n set ytics 1"
else:
yscale = "set autoscale y"
gnuplotFile.close()
gnuplotScript = \
"""#
set terminal png
set output '{dozorPlotFileName}'
set title '{title}'
set grid x2 y2
set xlabel 'Image number'
set x2label 'Angle (degrees)'
set y2label 'Resolution (A)'
set ylabel 'Number of spots / Dozor score (*10)'
set xtics {xtics} nomirror
set x2tics
set ytics nomirror
set y2tics
set xrange [{minImageNumber}:{maxImageNumber}]
set x2range [{minAngle}:{maxAngle}]
{yscale}
set y2range [{minResolution}:{maxResolution}]
set key below
plot '{dozorCsvFileName}' using 1:3 title 'Number of spots' axes x1y1 with points linetype rgb 'goldenrod' pointtype 7 pointsize 1.5, \
'{dozorCsvFileName}' using 1:4 title 'Dozor score' axes x1y1 with points linetype 3 pointtype 7 pointsize 1.5, \
'{dozorCsvFileName}' using 1:6 title 'Visible resolution' axes x1y2 with points linetype 1 pointtype 7 pointsize 1.5
""".format(title=self.template.replace("%04d", "####"),
dozorPlotFileName=dozorPlotFileName,
dozorCsvFileName=dozorCsvFileName,
minImageNumber=minImageNumber,
maxImageNumber=maxImageNumber,
minAngle=minAngle,
maxAngle=maxAngle,
minResolution=minResolution,
maxResolution=maxResolution,
xtics=xtics,
yscale=yscale,
)
pathGnuplotScript = os.path.join(self.getWorkingDirectory(), "gnuplot.sh")
data_file = open(pathGnuplotScript, "w")
data_file.write(gnuplotScript)
data_file.close()
oldCwd = os.getcwd()
os.chdir(self.getWorkingDirectory())
os.system("{0} {1}".format(self.gnuplot, pathGnuplotScript))
os.chdir(oldCwd)
self.dataOutput.dozorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), dozorPlotFileName)))
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = os.path.join(self.directory.replace("RAW_DATA", "PROCESSED_DATA"), "DozorPlot")
resultsDirectory = os.path.join(processDirectory, "results")
dozorPlotResultPath = os.path.join(resultsDirectory, dozorPlotFileName)
dozorCsvResultPath = os.path.join(resultsDirectory, dozorCsvFileName)
try:
if not os.path.exists(resultsDirectory):
os.makedirs(resultsDirectory, 0o755)
shutil.copy(os.path.join(self.getWorkingDirectory(), dozorPlotFileName), dozorPlotResultPath)
shutil.copy(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), dozorCsvResultPath)
except:
self.warning("Couldn't copy files to results directory: {0}".format(resultsDirectory))
try:
# Create paths on pyarch
dozorPlotPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorPlotResultPath)
dozorCsvPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorCsvResultPath)
if not os.path.exists(os.path.dirname(dozorPlotPyarchPath)):
os.makedirs(os.path.dirname(dozorPlotPyarchPath), 0o755)
shutil.copy(dozorPlotResultPath, dozorPlotPyarchPath)
shutil.copy(dozorCsvResultPath, dozorCsvPyarchPath)
# Upload to data collection
xsDataInputISPyBSetImageQualityIndicatorsPlot = XSDataInputISPyBSetImageQualityIndicatorsPlot()
xsDataInputISPyBSetImageQualityIndicatorsPlot.dataCollectionId = XSDataInteger(dataCollectionId)
xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsPlotPath = XSDataString(dozorPlotPyarchPath)
xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsCSVPath = XSDataString(dozorCsvPyarchPath)
EDPluginISPyBSetImageQualityIndicatorsPlot = self.loadPlugin("EDPluginISPyBSetImageQualityIndicatorsPlotv1_4")
EDPluginISPyBSetImageQualityIndicatorsPlot.dataInput = xsDataInputISPyBSetImageQualityIndicatorsPlot
EDPluginISPyBSetImageQualityIndicatorsPlot.executeSynchronous()
except:
self.warning("Couldn't copy files to pyarch: {0}".format(dozorPlotPyarchPath))
self.sendMessageToMXCuBE("Processing finished", "info")
self.setStatusToMXCuBE("Success")
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
if self.bIsEigerDetector:
xsDataResultCharacterisation = self.makeNumberOfImagesMultipleOf100(xsDataResultCharacterisation)
self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if self.createDNAFileDirectory(strPathToDNAFileDirectory):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation, strPathToDNAFileDirectory
)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
if self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory):
xsDataDictionaryLogFile = self.createOutputFileDictionary(
xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory
)
self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan is not None and xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(
XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory"
)
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
strBestWilsonPlotPyarchPath = None
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(
strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath)
)
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = (
self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
)
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin(
"EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath"
)
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
# Only for the ESRF:
if EDUtilsPath.isESRF():
# For EXI: create workflow entry with one workflow step
self.edPluginStoreWorkflow = self.loadPlugin(self.strPluginStoreWorkflow)
self.edPluginStoreWorkflowStep = self.loadPlugin(self.strPluginStoreWorkflowStep)
self.edPluginUpdateDataCollectionGroupWorkflowId = self.loadPlugin(
self.strUpdateDataCollectionGroupWorkflowId
)
xsDataISPyBWorkflow = XSDataISPyBWorkflow()
xsDataISPyBWorkflow.workflowType = XSDataString("Characterisation")
xsDataISPyBWorkflow.workflowTitle = XSDataString("Characterisation")
xsDataInputISPyBStoreWorkflow = XSDataInputISPyBStoreWorkflow()
xsDataInputISPyBStoreWorkflow.workflow = xsDataISPyBWorkflow
self.edPluginStoreWorkflow.dataInput = xsDataInputISPyBStoreWorkflow
self.edPluginStoreWorkflow.executeSynchronous()
if (
self.edPluginStoreWorkflow.dataOutput is not None
and self.edPluginStoreWorkflow.dataOutput.workflowId is not None
):
workflowId = self.edPluginStoreWorkflow.dataOutput.workflowId.value
# Update data collection group
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId = (
XSDataInputISPyBUpdateDataCollectionGroupWorkflowId()
)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.workflowId = XSDataInteger(workflowId)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileLocation = XSDataString(
os.path.dirname(self.xsDataFirstImage.path.value)
)
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId.fileName = XSDataString(
os.path.basename(self.xsDataFirstImage.path.value)
)
self.edPluginUpdateDataCollectionGroupWorkflowId.dataInput = (
xsDataInputISPyBUpdateDataCollectionGroupWorkflowId
)
self.edPluginUpdateDataCollectionGroupWorkflowId.executeSynchronous()
xsDataInputISPyBStoreWorkflowStep = XSDataInputISPyBStoreWorkflowStep()
xsDataInputISPyBStoreWorkflowStep.workflowId = XSDataInteger(workflowId)
xsDataInputISPyBStoreWorkflowStep.workflowStepType = XSDataString("Characterisation")
xsDataInputISPyBStoreWorkflowStep.status = XSDataString("Success")
if strBestWilsonPlotPyarchPath is not None:
xsDataInputISPyBStoreWorkflowStep.imageResultFilePath = XSDataString(
strBestWilsonPlotPyarchPath
)
xsDataInputISPyBStoreWorkflowStep.htmlResultFilePath = XSDataString(strPyArchPathToDNAFileDirectory)
if self.edPluginExecSimpleHTML.dataOutput is not None:
strResultFilePath = self.edPluginExecSimpleHTML.dataOutput.pathToJsonFile.path.value
# strPyarchResultFilePath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strResultFilePath)
xsDataInputISPyBStoreWorkflowStep.resultFilePath = XSDataString(strResultFilePath)
self.edPluginStoreWorkflowStep.dataInput = xsDataInputISPyBStoreWorkflowStep
self.edPluginStoreWorkflowStep.executeSynchronous()
def testCreatePyarchFilePath(self):
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"))
EDAssert.equal("/data/pyarch/id14eh2/mx415", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1"))
EDAssert.equal("/data/pyarch/id14eh2/mx415/20100212/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor/mx415/id14eh2/20100212/1/2"))
# Test with inhouse account...
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"))
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse"))
EDAssert.equal("/data/pyarch/id23eh2/opid232", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1"))
EDAssert.equal("/data/pyarch/id23eh2/opid232/20100525/1/2", EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2/inhouse/opid232/20100525/1/2"))
def storeResultsInISPyB(self, _strSubject, _strMessage):
strSubject = _strSubject
strMessage = _strMessage
xsDataResultCharacterisation = self.edPluginControlInterface.getDataOutput().getResultCharacterisation()
self.xsDataResultMXCuBE.setCharacterisationResult(xsDataResultCharacterisation)
xsDataResultControlISPyB = self.edPluginControlInterface.getDataOutput().getResultControlISPyB()
if xsDataResultControlISPyB != None:
self.xsDataResultMXCuBE.setScreeningId(xsDataResultControlISPyB.getScreeningId())
if xsDataResultCharacterisation != None:
self.xsDataResultMXCuBE.characterisationResult = xsDataResultCharacterisation
strPathCharacterisationResult = os.path.join(self.getWorkingDirectory(), "CharacterisationResult.xml")
xsDataResultCharacterisation.exportToFile(strPathCharacterisationResult)
self.xsDataResultMXCuBE.setListOfOutputFiles(XSDataString(strPathCharacterisationResult))
# For the moment, create "DNA" style output directory
strPathToDNAFileDirectory = self.createDNAFileDirectoryPath(xsDataResultCharacterisation)
xsDataDictionaryLogFile = None
if (self.createDNAFileDirectory(strPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPathToDNAFileDirectory)
strPyArchPathToDNAFileDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(strPathToDNAFileDirectory)
if (self.createDNAFileDirectory(strPyArchPathToDNAFileDirectory)):
xsDataDictionaryLogFile = self.createOutputFileDictionary(xsDataResultCharacterisation, strPyArchPathToDNAFileDirectory)
self.xsDataResultMXCuBE.setOutputFileDictionary(xsDataDictionaryLogFile)
if xsDataResultCharacterisation.getStatusMessage():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getStatusMessage().getValue()
if xsDataResultCharacterisation.getShortSummary():
strMessage += "\n\n"
strMessage += xsDataResultCharacterisation.getShortSummary().getValue()
self.sendEmail(strSubject, strMessage)
# Fix for bug EDNA-55 : If burning strategy EDNA2html shouldn't be run
bRunExecOutputHTML = False
xsDataInputMXCuBE = self.getDataInput()
xsDataDiffractionPlan = xsDataInputMXCuBE.getDiffractionPlan()
if xsDataDiffractionPlan.getStrategyOption() is not None:
strStrategyOption = xsDataDiffractionPlan.getStrategyOption().getValue()
if strStrategyOption.find("-DamPar") != -1:
bRunExecOutputHTML = False
if (self.edPluginExecOutputHTML is not None) and bRunExecOutputHTML:
self.edPluginExecOutputHTML.setDataInput(XSDataFile(XSDataString(strPathToDNAFileDirectory)), "dnaFileDirectory")
self.edPluginExecOutputHTML.execute()
# Fix for bug MXSUP-251: Put the BEST .par file in the EDNA characterisation root directory
xsDataIntegrationResult = xsDataResultCharacterisation.getIntegrationResult()
if xsDataIntegrationResult:
listXSDataIntegrationSubWedgeResult = xsDataIntegrationResult.getIntegrationSubWedgeResult()
for xsDataIntegrationSubWedgeResult in listXSDataIntegrationSubWedgeResult:
if xsDataIntegrationSubWedgeResult.getBestfilePar() is not None:
strBestfilePar = xsDataIntegrationSubWedgeResult.getBestfilePar().getValue()
# Put the file one directory above the mxCuBE v1.3 plugin working directory:
strDir = os.path.dirname(self.getWorkingDirectory())
strPath = os.path.join(strDir, "bestfile.par")
EDUtilsFile.writeFile(strPath, strBestfilePar)
break
# Execute plugin which creates a simple HTML page
self.executeSimpleHTML(xsDataResultCharacterisation)
# Upload the best wilson plot path to ISPyB
strBestWilsonPlotPath = EDHandlerXSDataISPyBv1_4.getBestWilsonPlotPath(xsDataResultCharacterisation)
if strBestWilsonPlotPath is not None and strPyArchPathToDNAFileDirectory is not None:
# Copy wilson path to Pyarch
strBestWilsonPlotPyarchPath = os.path.join(strPyArchPathToDNAFileDirectory, os.path.basename(strBestWilsonPlotPath))
if not os.path.exists(strBestWilsonPlotPyarchPath):
if not os.path.exists(os.path.dirname(strBestWilsonPlotPyarchPath)):
os.makedirs(os.path.dirname(strBestWilsonPlotPyarchPath), 755)
shutil.copy(strBestWilsonPlotPath, strBestWilsonPlotPyarchPath)
self.DEBUG("Best wilson pyarch path: %s " % strBestWilsonPlotPyarchPath)
if self.edPluginControlInterface.dataOutput.resultControlISPyB is not None:
xsDataInputISPyBSetBestWilsonPlotPath = XSDataInputISPyBSetBestWilsonPlotPath()
xsDataInputISPyBSetBestWilsonPlotPath.dataCollectionId = self.edPluginControlInterface.dataOutput.resultControlISPyB.dataCollectionId
xsDataInputISPyBSetBestWilsonPlotPath.bestWilsonPlotPath = XSDataString(strBestWilsonPlotPyarchPath)
edPluginSetBestWilsonPlotPath = self.loadPlugin("EDPluginISPyBSetBestWilsonPlotPathv1_4", "ISPyBSetBestWilsonPlotPath")
edPluginSetBestWilsonPlotPath.dataInput = xsDataInputISPyBSetBestWilsonPlotPath
edPluginSetBestWilsonPlotPath.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlCrystFELv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
#self.messenger.sendProcessingStatus(self.dataInput.dataCollectionId.value,
# "CrystFEL",
# "started")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.ispyb_retrieve_dc_plugin.dataInput = xsDataInputRetrieveDataCollection
self.ispyb_retrieve_dc_plugin.executeSynchronous()
ispybDataCollection = self.ispyb_retrieve_dc_plugin.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
#TODO PE2 has 7 digits others 5
template = ispybDataCollection.fileTemplate.replace("%07d", "####")
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
imageNoStart = ispybDataCollection.startImageNumber
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
try:
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 755)
except Exception as ex:
print ex
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
#TODO: enable when ready
"""
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 755)
"""
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 60
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.wait_first_image_plugin.size = XSDataInteger(minSizeFirst)
self.wait_first_image_plugin.dataInput = xsDataInputMXWaitFileFirst
self.wait_first_image_plugin.executeSynchronous()
if self.wait_first_image_plugin.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
self.baseName = "%s/crystfel_xgandalf" % self.resultsDirectory
self.streamFilename = "%s_%d.stream" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
self.hklFilename = "%s_%d.hkl" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
self.mtzFilename = "%s_%d.mtz" % (
self.baseName,
self.dataInput.dataCollectionId.value
)
# Prepare input to execution plugin
xsDataInputCrystFEL = XSDataInputCrystFEL()
# Check if imagesFullPath exists. If not then generate one
if self.dataInput.imagesFullPath is None:
imagesFullPath = "%s_images_fullpath.lst" % self.baseName
imagesFullFile = open(imagesFullPath, "w")
for index in range(imageNoEnd - imageNoStart + 1):
imagePath = os.path.join(directory, ispybDataCollection.fileTemplate % (index + 1))
imagesFullFile.write("%s\n" % imagePath)
imagesFullFile.close()
xsDataInputCrystFEL.imagesFullPath = XSDataString(imagesFullPath)
else:
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.geomFile = self.dataInput.geomFile
xsDataInputCrystFEL.cellFile = self.dataInput.cellFile
#TODO EDNA can generate the imagesFull path if id do not exist
xsDataInputCrystFEL.imagesFullPath = self.dataInput.imagesFullPath
xsDataInputCrystFEL.streamFile = XSDataString(self.streamFilename)
xsDataInputCrystFEL.hklFile = XSDataString(self.hklFilename)
xsDataInputCrystFEL.mtzFile = XSDataString(self.mtzFilename)
self.timeStart = time.localtime()
"""
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
self.autoProcIntegrationId, self.autoProcProgramId = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
"""
self.index_plugin.dataInput = xsDataInputCrystFEL
self.index_plugin.executeSynchronous()
if self.index_plugin.isFailure():
self.ERROR('indexamajig: Failed')
self.setFailure()
return
else:
self.screen('indexamajig: Finished')
self.process_hkl_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_even.dataInput = xsDataInputCrystFEL
self.partialator_plugin.dataInput = xsDataInputCrystFEL
self.post_process_plugin.dataInput = xsDataInputCrystFEL
self.process_hkl_plugin_odd.process_hkl_options += " --odd-only"
self.process_hkl_plugin_even.process_hkl_options += " --even-only"
self.process_hkl_plugin.executeSynchronous()
if self.process_hkl_plugin.isFailure():
self.ERROR('process_hkl: Failed')
self.setFailure()
return
else:
self.screen('process_hkl: Finished')
self.process_hkl_plugin_odd.dataInput.hklFile = XSDataString(self.hklFilename + "_o")
self.process_hkl_plugin_odd.executeSynchronous()
if self.process_hkl_plugin_odd.isFailure():
self.ERROR('process_hkl_odd: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_odd: Finished')
self.process_hkl_plugin_even.dataInput.hklFile = XSDataString(self.hklFilename + "_e")
self.process_hkl_plugin_even.executeSynchronous()
if self.process_hkl_plugin_even.isFailure():
self.ERROR('process_hkl_even: Failed')
self.setFailure()
return
else:
self.screen('process_hkl_even: Finished')
self.partialator_plugin.executeSynchronous()
if self.partialator_plugin.isFailure():
self.ERROR('partialator: Failed')
self.setFailure()
return
else:
self.screen('partialator: Finished')
self.post_process_plugin.executeSynchronous()
if self.post_process_plugin.isFailure():
self.ERROR('post_process: Failed')
self.setFailure()
return
else:
self.screen('post_process: Finished')
"""
self.edPluginExecCrystFELScale.dataInput = xsDataInputCrystFEL
self.edPluginExecCrystFELScale.executeSynchronous()
if self.edPluginExecCrystFELScale.isFailure():
self.ERROR('CrystFELScalev1_0: Failed')
self.setFailure()
return
else:
self.screen('CrystFELScalev1_0: Finished')
self.timeEnd = time.localtime()
"""
# Upload to ISPyB
"""
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')
configDef = None
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "#" * 5)
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.fromN is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.fromN.value
if self.dataInput.toN is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.toN.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
configDef = self.dataInput.configDef.path.value
directory = self.dataInput.dirN.path.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
if EDUtilsPath.isEMBL():
fileTemplate = template.replace("#####", "%05d")
else:
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory,
fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
if EDUtilsPath.isALBA():
_processDirectory = "_".join(pathToStartImage.split('_')[:-1])
from datetime import datetime
_id = datetime.now().strftime('%Y%m%d_%H%M%S')
self.resultsDirectory = os.path.join(_processDirectory,
"autoPROC_%s" % _id)
else:
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "autoPROC", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# The resultsDirectory is not used at ALBA (only pyarchDirectory)
if EDUtilsPath.isALBA():
self.resultsDirectory = None
# Determine pyarch prefix
if EDUtilsPath.isALBA():
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
listPrefix[-2])
else:
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60 # s
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart)
self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart)
self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart)
# Prepare input to execution plugin
xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCAnom.symm = self.dataInput.symm
xsDataInputAutoPROCAnom.cell = self.dataInput.cell
xsDataInputAutoPROCAnom.configDef = self.dataInput.configDef #XSDataFile(XSDataString(configDef))
# from future
# xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
# xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
# Added reprocess, low and high resolution limits for ControlAutoPROC
if self.doAnomAndNonanom:
# from future
# if self.doAnom:
# xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
# xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
# xsDataInputAutoPROCAnom.symm = self.dataInput.symm
# xsDataInputAutoPROCAnom.cell = self.dataInput.cell
# xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
# xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
# if self.doNoanom:
# Added 'doAnom' optional input
#<<<<<<< HEAD
# if self.doAnom:
# xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
# xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
# xsDataInputAutoPROCAnom.symm = self.dataInput.symm
# xsDataInputAutoPROCAnom.cell = self.dataInput.cell
# xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
# xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
# xsDataInputAutoPROCAnom.configDef = self.dataInput.configDef #XSDataFile(XSDataString(configDef))
# if self.doNoanom:
#=======
# xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
# xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
# xsDataInputAutoPROCAnom.symm = self.dataInput.symm
# xsDataInputAutoPROCAnom.cell = self.dataInput.cell
# xsDataInputAutoPROCAnom.configDef = self.dataInput.configDef #XSDataFile(XSDataString(configDef))
# if self.doAnomAndNonanom:
#>>>>>>> Add support for AutoPROC plugin
xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
xsDataInputAutoPROCNoanom.configDef = self.dataInput.configDef
# from future
# xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
# xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
# Added 'doAnom' optional input
#<<<<<<< HEAD
# xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
# xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
#=======
# xsDataInputAutoPROCNoanom.configDef = self.dataInput.configDef
#>>>>>>> Add support for AutoPROC plugin
xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
xsDataAutoPROCIdentifier.templateN = XSDataString(template)
xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
if self.doAnom:
xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
if self.doNoanom:
xsDataInputAutoPROCNoanom.addIdentifier(
xsDataAutoPROCIdentifier.copy())
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputAutoPROCAnom.masterH5 = XSDataFile(
XSDataString(masterFilePath))
if self.doNoanom:
xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(
XSDataString(masterFilePath))
timeStart = time.localtime()
if self.doAnom:
self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
self.edPluginExecAutoPROCAnom.execute()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
self.edPluginExecAutoPROCNoanom.execute()
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
timeEnd = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Upload to ISPyB
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False,
proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True,
proposal, timeStart, timeEnd)
if self.doAnomAndNonanom:
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False,
False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False,
True, proposal, timeStart, timeEnd)
def testCreatePyarchFilePath(self):
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"),
"/")
EDAssert.equal(None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"),
"/data")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/visitor"),
"/data/visitor")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2"), "/data/visitor/mx415/id14eh2")
EDAssert.equal(
"/data/pyarch/2010/id14eh2/mx415/20100212",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2/20100212"),
"/data/visitor/mx415/id14eh2/20100212")
EDAssert.equal(
"/data/pyarch/2010/id14eh2/mx415/20100212/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2/20100212/1"),
"/data/visitor/mx415/id14eh2/20100212/1")
EDAssert.equal(
"/data/pyarch/2010/id14eh2/mx415/20100212/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id14eh2/20100212/1/2"),
"/data/visitor/mx415/id14eh2/20100212/1/2")
# Test with inhouse account...
EDAssert.equal(None, EDHandlerESRFPyarchv1_0.createPyarchFilePath("/"),
"/")
EDAssert.equal(None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data"),
"/data")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath("/data/id23eh2"),
"/data/id23eh2")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse"), "/data/id23eh2/inhouse")
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232"),
"/data/id23eh2/inhouse/opid232")
EDAssert.equal(
"/data/pyarch/2010/id23eh2/opid232/20100525",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232/20100525"),
"/data/id23eh2/inhouse/opid232/20100525")
EDAssert.equal(
"/data/pyarch/2010/id23eh2/opid232/20100525/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232/20100525/1"),
"/data/id23eh2/inhouse/opid232/20100525/1")
EDAssert.equal(
"/data/pyarch/2010/id23eh2/opid232/20100525/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id23eh2/inhouse/opid232/20100525/1/2"),
"/data/id23eh2/inhouse/opid232/20100525/1/2")
EDAssert.equal(
"/data/pyarch/2014/id30a1/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
),
"/data/id30a1/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
)
# Visitor
EDAssert.equal(
None,
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3"), "/data/visitor/mx415/id30a3")
EDAssert.equal(
"/data/pyarch/2010/id30a3/mx415/20100212",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3/20100212"),
"/data/visitor/mx415/id30a3/20100212")
EDAssert.equal(
"/data/pyarch/2010/id30a3/mx415/20100212/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3/20100212/1"),
"/data/visitor/mx415/id30a3/20100212/1")
EDAssert.equal(
"/data/pyarch/2010/id30a3/mx415/20100212/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/visitor/mx415/id30a3/20100212/1/2"),
"/data/visitor/mx415/id30a3/20100212/1/2")
EDAssert.equal(
"/data/pyarch/2010/id30a3/opid232/20100525",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid232/20100525"),
"/data/id30a3/inhouse/opid232/20100525")
EDAssert.equal(
"/data/pyarch/2010/id30a3/opid232/20100525/1",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid232/20100525/1"),
"/data/id30a3/inhouse/opid232/20100525/1")
EDAssert.equal(
"/data/pyarch/2010/id30a3/opid232/20100525/1/2",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid232/20100525/1/2"),
"/data/id30a3/inhouse/opid232/20100525/1/2")
EDAssert.equal(
"/data/pyarch/2014/id30a3/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles",
EDHandlerESRFPyarchv1_0.createPyarchFilePath(
"/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
),
"/data/id30a3/inhouse/opid30a1/20140717/RAW_DATA/opid30a1_1_dnafiles"
)
