def input_from_params(self, data_collection, char_params):
edna_input = XSDataInputMXCuBE.parseString(self.edna_default_input)
if data_collection.id:
edna_input.setDataCollectionId(XSDataInteger(data_collection.id))
# Beam object
beam = edna_input.getExperimentalCondition().getBeam()
try:
transmission = HWR.beamline.transmission.get_value()
beam.setTransmission(XSDataDouble(transmission))
except AttributeError:
import traceback
logging.getLogger("HWR").debug(
"EDNACharacterisation. transmission not saved ")
logging.getLogger("HWR").debug(traceback.format_exc())
try:
wavelength = HWR.beamline.energy.get_wavelength()
beam.setWavelength(XSDataWavelength(wavelength))
except AttributeError:
pass
try:
beam.setFlux(XSDataFlux(HWR.beamline.flux.get_value()))
except AttributeError:
pass
try:
min_exp_time = self.collect_obj.detector_hwobj.get_exposure_time_limits(
)[0]
beam.setMinExposureTimePerImage(XSDataTime(min_exp_time))
except AttributeError:
pass
try:
beamsize = self.collect_obj.beam_info_hwobj.get_beam_size()
if None not in beamsize:
beam.setSize(
XSDataSize(
x=XSDataLength(float(beamsize[0])),
y=XSDataLength(float(beamsize[1])),
))
except AttributeError:
pass
# Optimization parameters
diff_plan = edna_input.getDiffractionPlan()
aimed_i_sigma = XSDataDouble(char_params.aimed_i_sigma)
aimed_completness = XSDataDouble(char_params.aimed_completness)
aimed_multiplicity = XSDataDouble(char_params.aimed_multiplicity)
aimed_resolution = XSDataDouble(char_params.aimed_resolution)
complexity = char_params.strategy_complexity
complexity = XSDataString(qme.STRATEGY_COMPLEXITY[complexity])
permitted_phi_start = XSDataAngle(char_params.permitted_phi_start)
_range = char_params.permitted_phi_end - char_params.permitted_phi_start
rotation_range = XSDataAngle(_range)
if char_params.aimed_i_sigma:
diff_plan.setAimedIOverSigmaAtHighestResolution(aimed_i_sigma)
if char_params.aimed_completness:
diff_plan.setAimedCompleteness(aimed_completness)
if char_params.use_aimed_multiplicity:
diff_plan.setAimedMultiplicity(aimed_multiplicity)
if char_params.use_aimed_resolution:
diff_plan.setAimedResolution(aimed_resolution)
diff_plan.setComplexity(complexity)
if char_params.use_permitted_rotation:
diff_plan.setUserDefinedRotationStart(permitted_phi_start)
diff_plan.setUserDefinedRotationRange(rotation_range)
# Vertical crystal dimension
sample = edna_input.getSample()
sample.getSize().setY(XSDataLength(char_params.max_crystal_vdim))
sample.getSize().setZ(XSDataLength(char_params.min_crystal_vdim))
# Radiation damage model
sample.setSusceptibility(XSDataDouble(char_params.rad_suscept))
sample.setChemicalComposition(None)
sample.setRadiationDamageModelBeta(XSDataDouble(char_params.beta /
1e6))
sample.setRadiationDamageModelGamma(
XSDataDouble(char_params.gamma / 1e6))
diff_plan.setForcedSpaceGroup(XSDataString(char_params.space_group))
# Characterisation type - Routine DC
if char_params.use_min_dose:
pass
if char_params.use_min_time:
time = XSDataTime(char_params.min_time)
diff_plan.setMaxExposureTimePerDataCollection(time)
# Account for radiation damage
if char_params.induce_burn:
self._modify_strategy_option(diff_plan, "-DamPar")
# Characterisation type - SAD
if char_params.opt_sad:
if char_params.auto_res:
diff_plan.setAnomalousData(XSDataBoolean(True))
else:
diff_plan.setAnomalousData(XSDataBoolean(False))
self._modify_strategy_option(diff_plan, "-SAD yes")
diff_plan.setAimedResolution(XSDataDouble(char_params.sad_res))
else:
diff_plan.setAnomalousData(XSDataBoolean(False))
# Data set
data_set = XSDataMXCuBEDataSet()
acquisition_parameters = data_collection.acquisitions[
0].acquisition_parameters
path_template = data_collection.acquisitions[0].path_template
path_str = os.path.join(path_template.directory,
path_template.get_image_file_name())
for img_num in range(int(acquisition_parameters.num_images)):
image_file = XSDataFile()
path = XSDataString()
path.setValue(path_str % (img_num + 1))
image_file.setPath(path)
data_set.addImageFile(image_file)
edna_input.addDataSet(data_set)
edna_input.process_directory = path_template.process_directory
return edna_input
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_6.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist" % dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist" % inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
self.addExecutiveSummaryLine(
"Got only one frame ... nothing to merge !!!")
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], self.lstInput[1]])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.addExecutiveSummaryLine("WARNING: " +
self.strRadiationDamage)
self.addExecutiveSummaryLine("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
# outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer
is not None) and (self.__class__.lastSample is not None):
self.__edPluginExecAutoSub = self.loadPlugin(
self.__strControlledPluginAutoSub)
base = "_".join(
os.path.basename(
self.__class__.lastSample.path.value).split("_")[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile),
base + "_" + suff)
self.xsdSubtractedCurve = XSDataFile(XSDataString(sub))
#self.curves.append(xsdSubtractedCurve)
self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub(
sampleCurve=self.__class__.lastSample,
buffers=[
self.__class__.lastBuffer, self.dataInput.mergedCurve
],
subtractedCurve=self.xsdSubtractedCurve)
self.__edPluginExecAutoSub.connectSUCCESS(
self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(
self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
if self.isFailure():
return
self.__edPluginSaxsAnalysis = self.loadPlugin(
self.__strControlledPluginSaxsAnalysis)
self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis(
scatterCurve=self.xsdSubtractedCurve,
autoRg=self.autoRg,
graphFormat=XSDataString("png"))
self.__edPluginSaxsAnalysis.connectSUCCESS(
self.doSuccessSaxsAnalysis)
self.__edPluginSaxsAnalysis.connectFAILURE(
self.doFailureSaxsAnalysis)
self.__edPluginSaxsAnalysis.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
#self.__class__.lastSample = None #Information neededfor transfer to ISPyB
self.forgetLastSample = True
else:
self.__class__.lastSample = self.dataInput.mergedCurve
if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID:
self.addExecutiveSummaryLine("Registering to ISPyB")
self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged))
self.__class__.dictFrames[self.dataInput.mergedCurve] = {
'averaged': self.lstMerged,
'discarded': self.lstDiscarded
}
self.__edPluginSaxsISPyB = self.loadPlugin(
self.__strControlledPluginSaxsISPyB)
if len(self.lstInput) > 1:
frameAverage = XSDataInteger(len(self.lstInput))
frameMerged = XSDataInteger(len(self.lstMerged))
else:
frameMerged = frameAverage = XSDataInteger(1)
self.curves = [XSDataFile(i.path) for i in self.lstInput]
self.discardedCurves = [
XSDataFile(i.path) for i in self.lstDiscarded
]
self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged]
averageFilePath = None
if self.strMergedFile is not None:
averageFilePath = XSDataFile(XSDataString(self.strMergedFile))
self.sampleFrames = self.getAveragedFrameByFilename(
self.__class__.lastSample)
lastSample = None
if self.__class__.lastSample is not None:
lastSample = self.__class__.lastSample
subtractedCurve = None
if self.xsdSubtractedCurve is not None:
subtractedCurve = self.xsdSubtractedCurve
xsdin = XSDataInputBioSaxsISPyBv1_0(
sample=self.dataInput.sample,
autoRg=self.autoRg,
gnom=self.gnom,
volume=self.volume,
frameAverage=frameAverage,
frameMerged=frameMerged,
curves=self.curves,
discardedFrames=self.discardedCurves,
averagedFrames=self.mergedCurves,
averageFilePath=averageFilePath,
bufferFrames=self.bufferFrames,
sampleFrames=self.sampleFrames,
bestBuffer=self.xsBestBuffer,
averageSample=lastSample,
scatterPlot=self.xsScatterPlot,
guinierPlot=self.xsGuinierPlot,
kratkyPlot=self.xsKratkyPlot,
densityPlot=self.xsDensityPlot,
subtractedFilePath=subtractedCurve
# destination=self.dataInput.sample.ispybDestination #duplicate, already in sample
)
self.__edPluginSaxsISPyB.dataInput = xsdin
self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB)
self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB)
self.__edPluginSaxsISPyB.execute()
if self.forgetLastSample:
#Also redefine dictionary to contain the buffer just processed?
self.__class__.lastSample = None
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
os.makedirs(self.pyarchDirectory, 0o755)
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(masterFilePath)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(masterFilePath)))
else:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
if self.doAnomAndNonanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doAnomAndNonanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doAnomAndNonanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doAnomAndNonanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("*** EDPluginExecReadImageHeaderADSCv10.process")
xsDataInputReadImageHeader = self.getDataInput()
xsDataFile = xsDataInputReadImageHeader.getImage()
strPath = xsDataFile.getPath().getValue()
strAbsolutePath = os.path.abspath(strPath)
dictHeader = self.readHeaderADSC(strPath)
if (dictHeader is None):
strErrorMessage = "EDPluginExecReadImageHeaderADSCv10.process : error when reading header from %s" % strAbsolutePath
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
else:
xsDataExperimentalCondition = XSDataExperimentalCondition()
xsDataDetector = XSDataDetector()
xsDataDetector.setBeamPositionX(
XSDataLength(float(dictHeader["BEAM_CENTER_X"])))
xsDataDetector.setBeamPositionY(
XSDataLength(float(dictHeader["BEAM_CENTER_Y"])))
xsDataDetector.setDistance(
XSDataLength(float(dictHeader["DISTANCE"])))
fPixelSize = float(dictHeader["PIXEL_SIZE"])
xsDataDetector.setPixelSizeX(XSDataLength(fPixelSize))
xsDataDetector.setPixelSizeY(XSDataLength(fPixelSize))
if "TWOTHETA" in dictHeader.keys():
xsDataDetector.setTwoTheta(
XSDataAngle(float(dictHeader["TWOTHETA"])))
xsDataDetector.setNumberBytesInHeader(
XSDataInteger(float(dictHeader["HEADER_BYTES"])))
xsDataDetector.setSerialNumber(
XSDataString(dictHeader["DETECTOR_SN"]))
xsDataDetector.setNumberPixelX(
XSDataInteger(int(dictHeader["SIZE1"])))
xsDataDetector.setNumberPixelY(
XSDataInteger(int(dictHeader["SIZE2"])))
xsDataDetector.setBin(XSDataString(dictHeader["BIN"]))
xsDataDetector.setDataType(XSDataString(dictHeader["TYPE"]))
xsDataDetector.setByteOrder(XSDataString(dictHeader["BYTE_ORDER"]))
if "CCD_IMAGE_SATURATION" in dictHeader.keys():
xsDataDetector.setImageSaturation(
XSDataInteger(int(dictHeader["CCD_IMAGE_SATURATION"])))
# Determine type of detector...
iNoPixelsX = xsDataDetector.getNumberPixelX().getValue()
iNoPixelsY = xsDataDetector.getNumberPixelY().getValue()
if (iNoPixelsX == 2304 and iNoPixelsY == 2304):
xsDataDetector.setName(XSDataString("ADSC Q4"))
xsDataDetector.setType(XSDataString("q4"))
elif (iNoPixelsX == 1152 and iNoPixelsY == 1152):
xsDataDetector.setName(XSDataString("ADSC Q4 bin 2x2"))
xsDataDetector.setType(XSDataString("q4-2x"))
elif (iNoPixelsX == 4096 and iNoPixelsY == 4096):
xsDataDetector.setName(XSDataString("ADSC Q210"))
xsDataDetector.setType(XSDataString("q210"))
elif (iNoPixelsX == 2048 and iNoPixelsY == 2048):
xsDataDetector.setName(XSDataString("ADSC Q210 bin 2x2"))
xsDataDetector.setType(XSDataString("q210-2x"))
elif (iNoPixelsX == 6144 and iNoPixelsY == 6144):
xsDataDetector.setName(XSDataString("ADSC Q315"))
xsDataDetector.setType(XSDataString("q315"))
elif (iNoPixelsX == 3072 and iNoPixelsY == 3072):
xsDataDetector.setName(XSDataString("ADSC Q315 bin 2x2"))
xsDataDetector.setType(XSDataString("q315-2x"))
else:
strErrorMessage = EDMessage.ERROR_DATA_HANDLER_02 % (
"EDPluginExecReadImageHeaderADSCv10.process",
"Unknown detector type")
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
xsDataExperimentalCondition.setDetector(xsDataDetector)
# Beam object
xsDataBeam = XSDataBeam()
xsDataBeam.setWavelength(
XSDataWavelength(float(dictHeader["WAVELENGTH"])))
xsDataBeam.setExposureTime(XSDataTime(float(dictHeader["TIME"])))
xsDataExperimentalCondition.setBeam(xsDataBeam)
# Goniostat object
xsDataGoniostat = XSDataGoniostat()
fRotationAxisStart = float(dictHeader["OSC_START"])
fOscillationWidth = float(dictHeader["OSC_RANGE"])
xsDataGoniostat.setRotationAxisStart(
XSDataAngle(fRotationAxisStart))
xsDataGoniostat.setRotationAxisEnd(
XSDataAngle(fRotationAxisStart + fOscillationWidth))
xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth))
strRotationAxis = None
if ("AXIS" in dictHeader.keys()):
strRotationAxis = dictHeader["AXIS"]
elif ("OSC_AXIS" in dictHeader.keys()):
strRotationAxis = dictHeader["OSC_AXIS"]
else:
strErrorMessage = "EDPluginExecReadImageHeaderADSCv10.process : Neither AXIS nor OSC_AXIS header item found."
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
xsDataGoniostat.setRotationAxis(XSDataString(strRotationAxis))
xsDataExperimentalCondition.setGoniostat(xsDataGoniostat)
# Create the image object
xsDataImage = XSDataImage()
xsDataImage.setPath(XSDataString(strAbsolutePath))
xsDataImage.setDate(XSDataString(dictHeader["DATE"]))
strFileName = os.path.basename(strPath)
iImageNumber = EDUtilsImage.getImageNumber(strFileName)
xsDataImage.setNumber(XSDataInteger(iImageNumber))
xsDataSubWedge = XSDataSubWedge()
xsDataSubWedge.setExperimentalCondition(
xsDataExperimentalCondition)
xsDataSubWedge.addImage(xsDataImage)
self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader()
self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
EDUtilsParallel.initializeNbThread()
# Check batch size
if self.dataInput.batchSize is None:
batchSize = 1
else:
batchSize = self.dataInput.batchSize.value
self.screen(
"Image quality indicators batch size: {0}".format(batchSize))
# Check if we should do distlSignalStrength:
bDoDistlSignalStrength = True
if self.dataInput.doDistlSignalStrength is not None:
if not self.dataInput.doDistlSignalStrength.value:
bDoDistlSignalStrength = False
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Check if fast mesh (for HDF5)
isFastMesh = False
if self.dataInput.fastMesh:
isFastMesh = self.dataInput.fastMesh.value
# Loop through all the incoming reference images
if len(self.dataInput.image) == 0:
directory = self.dataInput.directory.path.value
template = self.dataInput.template.value
startNo = self.dataInput.startNo.value
endNo = self.dataInput.endNo.value
listXSDataImage = []
for index in range(startNo, endNo + 1):
imageName = template.replace("####", "{0:04d}".format(index))
imagePath = os.path.join(directory, imageName)
xsDataImage = XSDataImage(path=XSDataString(imagePath),
number=XSDataInteger(index))
listXSDataImage.append(xsDataImage)
else:
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
)
listPluginDistl = []
listPluginDozor = []
listOfImagesInBatch = []
listOfAllBatches = []
indexBatch = 0
listH5FilePath = []
# Process data in batches
for xsDataImage in listXSDataImage:
listOfImagesInBatch.append(xsDataImage.copy())
if len(listOfImagesInBatch) == batchSize:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
if len(listOfImagesInBatch) > 0:
listOfAllBatches.append(listOfImagesInBatch)
listOfImagesInBatch = []
# Loop over batches
for listOfImagesInBatch in listOfAllBatches:
# First wait for images
for image in listOfImagesInBatch:
strPathToImage = image.path.value
# If Eiger, just wait for the h5 file
if strPathToImage.endswith(".h5"):
h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(
strPathToImage,
batchSize=batchSize,
isFastMesh=isFastMesh)
# print(h5FilePath)
# print(hdf5ImageNumber)
if not h5DataFilePath in listH5FilePath:
self.screen(
"ID30a3 Eiger data, waiting for master and data files..."
)
listH5FilePath.append(h5DataFilePath)
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(
XSDataString(h5DataFilePath))
xsDataInputMXWaitFile.setSize(
XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.screen(
"Waiting for file {0}".format(h5DataFilePath))
self.DEBUG("Wait file timeOut set to %f" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(
xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
# hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
time.sleep(1)
if not os.path.exists(h5DataFilePath):
strError = "Time-out while waiting for image %s" % h5DataFilePath
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
if not os.path.exists(strPathToImage):
# self.screen("Waiting for file {0}".format(strPathToImage))
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(
XSDataString(strPathToImage))
xsDataInputMXWaitFile.setSize(
XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.screen("Wait file timeOut set to %.0f s" %
self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(
xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(strPathToImage):
strError = "Time-out while waiting for image %s" % strPathToImage
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
if not self.isFailure():
strPathToFirstImage = listOfImagesInBatch[0].path.value
if strPathToImage.endswith(".h5"):
indexLoop = 1
continueLoop = True
while continueLoop:
directory = os.path.dirname(strPathToFirstImage)
firstImage = EDUtilsImage.getImageNumber(
listOfImagesInBatch[0].path.value)
lastImage = EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)
xsDataInputH5ToCBF = XSDataInputH5ToCBF()
xsDataInputH5ToCBF.hdf5File = XSDataFile(
listOfImagesInBatch[0].path)
xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
xsDataInputH5ToCBF.startImageNumber = XSDataInteger(
firstImage)
xsDataInputH5ToCBF.endImageNumber = XSDataInteger(
lastImage)
xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(
XSDataString(directory))
edPluginH5ToCBF = self.loadPlugin(
"EDPluginH5ToCBFv1_1")
edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
edPluginH5ToCBF.execute()
edPluginH5ToCBF.synchronize()
outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
if outputCBFFileTemplate is not None:
lastCbfFile = outputCBFFileTemplate.path.value.replace(
"######", "{0:06d}".format(
EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)))
strPathToImage = os.path.join(
directory, lastCbfFile)
# print(cbfFile.path.value)
if os.path.exists(strPathToImage):
# Rename all images
for image in listOfImagesInBatch:
image.path.value = image.path.value.replace(
".h5", ".cbf")
imageNumber = EDUtilsImage.getImageNumber(
image.path.value)
oldPath = os.path.join(
directory,
outputCBFFileTemplate.path.value.
replace("######",
"{0:06d}".format(imageNumber)))
newPath = os.path.join(
directory,
outputCBFFileTemplate.path.value.
replace("######",
"{0:04d}".format(imageNumber)))
os.rename(oldPath, newPath)
lastCbfFile = outputCBFFileTemplate.path.value.replace(
"######", "{0:04d}".format(
EDUtilsImage.getImageNumber(
listOfImagesInBatch[-1].path.value)
))
strPathToImage = os.path.join(
directory, lastCbfFile)
self.screen(
"Image has been converted to CBF file: {0}"
.format(strPathToImage))
continueLoop = False
# print(continueLoop)
if continueLoop:
self.screen(
"Still waiting for converting to CBF file: {0}"
.format(strPathToImage))
indexLoop += 1
time.sleep(5)
if indexLoop > 10:
continueLoop = False
for image in listOfImagesInBatch:
strPathToImage = image.path.value
# Check if we should run distl.signalStrength
xsDataImageNew = XSDataImage(XSDataString(strPathToImage))
xsDataImageNew.number = XSDataInteger(
EDUtilsImage.getImageNumber(image.path.value))
edPluginPluginExecImageQualityIndicator = None
if bDoDistlSignalStrength:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(
strPluginName)
self.listPluginExecImageQualityIndicator.append(
edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
)
xsDataInputDistlSignalStrength.setReferenceImage(
xsDataImageNew)
edPluginPluginExecImageQualityIndicator.setDataInput(
xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listPluginDistl.append(
(xsDataImageNew.copy(),
edPluginPluginExecImageQualityIndicator))
edPluginControlDozor = self.loadPlugin(
self.strPluginNameControlDozor, "ControlDozor_{0}".format(
os.path.splitext(
os.path.basename(strPathToFirstImage))[0]))
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listOfImagesInBatch:
xsDataInputControlDozor.addImage(XSDataFile(image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(
len(listOfImagesInBatch))
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.execute()
listPluginDozor.append(
(edPluginControlDozor, list(listOfImagesInBatch)))
if not self.isFailure():
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
)
for (xsDataImage,
edPluginPluginExecImageQualityIndicator) in listPluginDistl:
xsDataImageQualityIndicators = XSDataImageQualityIndicators()
xsDataImageQualityIndicators.image = xsDataImage.copy()
if edPluginPluginExecImageQualityIndicator is not None:
edPluginPluginExecImageQualityIndicator.synchronize()
if edPluginPluginExecImageQualityIndicator.dataOutput is not None:
if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
xsDataImageQualityIndicators)
for (edPluginControlDozor, listBatch) in listPluginDozor:
edPluginControlDozor.synchronize()
# Check that we got at least one result
if len(edPluginControlDozor.dataOutput.imageDozor) == 0:
# Run the dozor plugin again, this time synchronously
firstImage = os.path.basename(listBatch[0].path.value)
lastImage = os.path.basename(listBatch[-1].path.value)
self.screen(
"No dozor results! Re-executing Dozor for images {0} to {1}"
.format(firstImage, lastImage))
time.sleep(5)
edPluginControlDozor = self.loadPlugin(
self.strPluginNameControlDozor,
"ControlDozor_reexecution_{0}".format(
os.path.splitext(firstImage)[0]))
xsDataInputControlDozor = XSDataInputControlDozor()
for image in listBatch:
xsDataInputControlDozor.addImage(XSDataFile(
image.path))
xsDataInputControlDozor.batchSize = XSDataInteger(
batchSize)
edPluginControlDozor.dataInput = xsDataInputControlDozor
edPluginControlDozor.executeSynchronous()
for imageDozor in edPluginControlDozor.dataOutput.imageDozor:
for xsDataImageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if xsDataImageQualityIndicators.image.path.value == imageDozor.image.path.value:
xsDataImageQualityIndicators.dozor_score = imageDozor.mainScore
xsDataImageQualityIndicators.dozorSpotFile = imageDozor.spotFile
if imageDozor.spotFile is not None:
if os.path.exists(
imageDozor.spotFile.path.value):
numpyArray = numpy.loadtxt(
imageDozor.spotFile.path.value,
skiprows=3)
xsDataImageQualityIndicators.dozorSpotList = XSDataString(
base64.b64encode(
numpyArray.tostring()))
xsDataImageQualityIndicators.addDozorSpotListShape(
XSDataInteger(numpyArray.shape[0]))
if len(numpyArray.shape) > 1:
xsDataImageQualityIndicators.addDozorSpotListShape(
XSDataInteger(numpyArray.shape[1]))
xsDataImageQualityIndicators.dozorSpotsIntAver = imageDozor.spotsIntAver
xsDataImageQualityIndicators.dozorSpotsResolution = imageDozor.spotsResolution
xsDataImageQualityIndicators.dozorVisibleResolution = imageDozor.visibleResolution
if self.xsDataResultControlImageQualityIndicators.inputDozor is None:
if edPluginControlDozor.dataOutput.inputDozor is not None:
self.xsDataResultControlImageQualityIndicators.inputDozor = XSDataDozorInput(
).parseString(
edPluginControlDozor.dataOutput.
inputDozor.marshal())
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.dataInput.doUploadToIspyb is not None and self.dataInput.doUploadToIspyb.value:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
# Check that we have dozor_score from all images:
has_dozor_score = True
for imageQualityIndicators in self.xsDataResultControlImageQualityIndicators.imageQualityIndicators:
if imageQualityIndicators.dozor_score is None:
has_dozor_score = False
if has_dozor_score:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators:
imageQualityIndicators.dozor_score.value)
else:
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators:
imageQualityIndicators.totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[
-5:]:
if xsDataResultControlImageQualityIndicator.dozor_score.value > 1:
xsDataInputReadImageHeader = XSDataInputReadImageHeader(
)
xsDataInputReadImageHeader.image = XSDataFile(
xsDataResultControlImageQualityIndicator.image.path
)
self.edPluginReadImageHeader = self.loadPlugin(
self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
edPluginLabelitIndexing = self.loadPlugin(
self.strIndexingLabelitPluginName)
xsDataInputLabelitIndexing = XSDataInputLabelitIndexing(
)
xsDataInputLabelitIndexing.image.append(
XSDataImage(
xsDataResultControlImageQualityIndicator.
image.path))
edPluginLabelitIndexing.setDataInput(
xsDataInputLabelitIndexing)
self.listPluginLabelit.append([
edPluginLabelitIndexing,
xsDataResultControlImageQualityIndicator
])
edPluginLabelitIndexing.execute()
for tupleLabelit in self.listPluginLabelit:
edPluginLabelitIndexing = tupleLabelit[0]
xsDataResultControlImageQualityIndicator = tupleLabelit[1]
edPluginLabelitIndexing.synchronize()
if not edPluginLabelitIndexing.isFailure(
) and edPluginLabelitIndexing.dataOutput is not None:
xsDataResultLabelitIndexing = edPluginLabelitIndexing.getDataOutput(
)
xsDataIndexingResult = EDHandlerXSDataPhenixv1_1.generateXSDataIndexingResult(
xsDataResultLabelitIndexing)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalCondition':
obj_ = XSDataExperimentalCondition()
obj_.build(child_)
self.setExperimentalCondition(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'diffractionPlan':
obj_ = XSDataDiffractionPlan()
obj_.build(child_)
self.setDiffractionPlan(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'sample':
obj_ = XSDataSampleCrystalMM()
obj_.build(child_)
self.setSample(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'imagePath':
obj_ = XSDataImage()
obj_.build(child_)
self.imagePath.append(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'flux':
obj_ = XSDataFloat()
obj_.build(child_)
self.setFlux(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'minExposureTimePerImage':
obj_ = XSDataTime()
obj_.build(child_)
self.setMinExposureTimePerImage(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSize':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSize(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSizeX':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSizeX(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSizeY':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSizeY(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'templateMode':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setTemplateMode(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'generatedTemplateFile':
obj_ = XSDataFile()
obj_.build(child_)
self.setGeneratedTemplateFile(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'resultsFilePath':
obj_ = XSDataFile()
obj_.build(child_)
self.setResultsFilePath(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamPosX':
obj_ = XSDataFloat()
obj_.build(child_)
self.setBeamPosX(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamPosY':
obj_ = XSDataFloat()
obj_.build(child_)
self.setBeamPosY(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'wavelength':
obj_ = XSDataWavelength()
obj_.build(child_)
self.setWavelength(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'transmission':
obj_ = XSDataDouble()
obj_.build(child_)
self.setTransmission(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'dataCollectionId':
obj_ = XSDataInteger()
obj_.build(child_)
self.setDataCollectionId(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'shortComments':
obj_ = XSDataString()
obj_.build(child_)
self.setShortComments(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'comments':
obj_ = XSDataString()
obj_.build(child_)
self.setComments(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'inputCharacterisation':
obj_ = XSDataInputCharacterisation()
obj_.build(child_)
self.setInputCharacterisation(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'token':
obj_ = XSDataString()
obj_.build(child_)
self.setToken(obj_)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlAutoPROCv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace("%05d", "#" * 5)
elif EDUtilsPath.isMAXIV():
template = ispybDataCollection.fileTemplate
else:
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if self.dataInput.fromN is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.fromN.value
if self.dataInput.toN is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.toN.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.path.value
template = self.dataInput.templateN.value
imageNoStart = self.dataInput.fromN.value
imageNoEnd = self.dataInput.toN.value
if EDUtilsPath.isEMBL():
fileTemplate = template.replace("#####", "%05d")
else:
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
if EDUtilsPath.isALBA():
_processDirectory = "_".join(pathToStartImage.split('_')[:-1])
from datetime import datetime
_id = datetime.now().strftime('%Y%m%d_%H%M%S')
self.resultsDirectory = os.path.join(_processDirectory, "autoPROC_%s" % _id)
else:
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"autoPROC", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# The resultsDirectory is not used at ALBA (only pyarchDirectory)
if EDUtilsPath.isALBA():
self.resultsDirectory = None
# Determine pyarch prefix
if EDUtilsPath.isALBA():
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_{1}".format("_".join(listPrefix[:-2]),
listPrefix[-2])
else:
listPrefix = template.split("_")
self.pyarchPrefix = "ap_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isMAXIV():
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
if EDUtilsPath.isEMBL() or EDUtilsPath.isMAXIV():
fWaitFileTimeout = 60 # s
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=True,
timeStart=self.timeStart)
self.autoProcIntegrationIdAnomStaraniso, self.autoProcProgramIdAnomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgram,
isAnom=False,
timeStart=self.timeStart)
self.autoProcIntegrationIdNoanomStaraniso, self.autoProcProgramIdNoanomStaraniso = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingProgramStaraniso,
isAnom=False,
timeStart=self.timeStart)
# Prepare input to execution plugin
if self.doAnom:
xsDataInputAutoPROCAnom = XSDataInputAutoPROC()
xsDataInputAutoPROCAnom.anomalous = XSDataBoolean(True)
xsDataInputAutoPROCAnom.symm = self.dataInput.symm
xsDataInputAutoPROCAnom.cell = self.dataInput.cell
xsDataInputAutoPROCAnom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCAnom.highResolutionLimit = self.dataInput.highResolutionLimit
if self.doNoanom:
xsDataInputAutoPROCNoanom = XSDataInputAutoPROC()
xsDataInputAutoPROCNoanom.anomalous = XSDataBoolean(False)
xsDataInputAutoPROCNoanom.symm = self.dataInput.symm
xsDataInputAutoPROCNoanom.cell = self.dataInput.cell
xsDataInputAutoPROCNoanom.lowResolutionLimit = self.dataInput.lowResolutionLimit
xsDataInputAutoPROCNoanom.highResolutionLimit = self.dataInput.highResolutionLimit
xsDataAutoPROCIdentifier = XSDataAutoPROCIdentifier()
xsDataAutoPROCIdentifier.idN = XSDataString(identifier)
xsDataAutoPROCIdentifier.dirN = XSDataFile(XSDataString(directory))
xsDataAutoPROCIdentifier.templateN = XSDataString(template)
xsDataAutoPROCIdentifier.fromN = XSDataInteger(imageNoStart)
xsDataAutoPROCIdentifier.toN = XSDataInteger(imageNoEnd)
if self.doAnom:
xsDataInputAutoPROCAnom.addIdentifier(xsDataAutoPROCIdentifier)
if self.doNoanom:
xsDataInputAutoPROCNoanom.addIdentifier(xsDataAutoPROCIdentifier.copy())
if isH5:
masterFilePath = os.path.join(directory,
self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputAutoPROCAnom.masterH5 = XSDataFile(XSDataString(masterFilePath))
if self.doNoanom:
xsDataInputAutoPROCNoanom.masterH5 = XSDataFile(XSDataString(masterFilePath))
timeStart = time.localtime()
if self.doAnom:
self.edPluginExecAutoPROCAnom.dataInput = xsDataInputAutoPROCAnom
self.edPluginExecAutoPROCAnom.execute()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.dataInput = xsDataInputAutoPROCNoanom
self.edPluginExecAutoPROCNoanom.execute()
if self.doAnom:
self.edPluginExecAutoPROCAnom.synchronize()
if self.doNoanom:
self.edPluginExecAutoPROCNoanom.synchronize()
timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCAnom, True, True, proposal, timeStart, timeEnd)
if self.doNoanom:
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, False, proposal, timeStart, timeEnd)
self.uploadToISPyB(self.edPluginExecAutoPROCNoanom, False, True, proposal, timeStart, timeEnd)
def process(self, _edPlugin=None):
"""
Executes the execution plugins
"""
EDPluginControl.process(self, _edPlugin)
self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process")
EDUtilsParallel.initializeNbThread()
# Check if we should do indexing:
bDoIndexing = False
if self.dataInput.doIndexing is not None:
if self.dataInput.doIndexing.value:
bDoIndexing = True
# Loop through all the incoming reference images
listXSDataImage = self.dataInput.image
xsDataInputMXWaitFile = XSDataInputMXWaitFile()
self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
)
listPlugin = []
for xsDataImage in listXSDataImage:
self.edPluginMXWaitFile = self.loadPlugin(
self.strPluginMXWaitFileName)
xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
xsDataInputMXWaitFile.setTimeOut(
XSDataTime(self.fMXWaitFileTimeOut))
self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
self.edPluginMXWaitFile.executeSynchronous()
if not os.path.exists(xsDataImage.path.value):
strError = "Time-out while waiting for image %s" % xsDataImage.path.value
self.error(strError)
self.addErrorMessage(strError)
self.setFailure()
else:
if self.bUseThinClient:
strPluginName = self.strPluginNameThinClient
else:
strPluginName = self.strPluginName
edPluginPluginExecImageQualityIndicator = self.loadPlugin(
strPluginName)
listPlugin.append(edPluginPluginExecImageQualityIndicator)
self.listPluginExecImageQualityIndicator.append(
edPluginPluginExecImageQualityIndicator)
xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
)
xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
edPluginPluginExecImageQualityIndicator.setDataInput(
xsDataInputDistlSignalStrength)
edPluginPluginExecImageQualityIndicator.execute()
listIndexing = []
# Synchronize all image quality indicator plugins and upload to ISPyB
xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
)
for edPluginPluginExecImageQualityIndicator in listPlugin:
edPluginPluginExecImageQualityIndicator.synchronize()
xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \
edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
xsDataImageQualityIndicators)
xsDataISPyBImageQualityIndicators = \
XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
xsDataISPyBImageQualityIndicators)
# print xsDataInputStoreListOfImageQualityIndicators.marshal()
if self.bDoISPyBUpload:
self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
self.edPluginISPyB.execute()
#
if bDoIndexing:
# Find the 5 most intensive images (TIS):
listImage = []
listSorted = sorted(
self.xsDataResultControlImageQualityIndicators.
imageQualityIndicators,
key=lambda imageQualityIndicators: imageQualityIndicators.
totalIntegratedSignal.value)
for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = XSDataFile(
xsDataResultControlImageQualityIndicator.image.path)
self.edPluginReadImageHeader = self.loadPlugin(
self.strPluginReadImageHeaderName)
self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
self.edPluginReadImageHeader.executeSynchronous()
xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
if xsDataResultReadImageHeader is not None:
xsDataSubWedge = xsDataResultReadImageHeader.subWedge
self.xsDataCollection = XSDataCollection()
self.xsDataCollection.addSubWedge(xsDataSubWedge)
xsDataIndexingInput = XSDataIndexingInput()
xsDataIndexingInput.setDataCollection(
self.xsDataCollection)
xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(
xsDataIndexingInput)
edPluginMOSFLMIndexing = self.loadPlugin(
self.strIndexingMOSFLMPluginName)
self.listPluginMOSFLM.append([
edPluginMOSFLMIndexing,
xsDataResultControlImageQualityIndicator
])
edPluginMOSFLMIndexing.setDataInput(
xsDataMOSFLMIndexingInput)
edPluginMOSFLMIndexing.execute()
for tupleMOSFLM in self.listPluginMOSFLM:
edPluginMOSFLMIndexing = tupleMOSFLM[0]
xsDataResultControlImageQualityIndicator = tupleMOSFLM[1]
edPluginMOSFLMIndexing.synchronize()
if not edPluginMOSFLMIndexing.isFailure():
xsDataMOSFLMOutput = edPluginMOSFLMIndexing.dataOutput
xsDataIndexingResult = EDHandlerXSDataMOSFLMv10.generateXSDataIndexingResult(
xsDataMOSFLMOutput)
selectedSolution = xsDataIndexingResult.selectedSolution
if selectedSolution is not None:
xsDataResultControlImageQualityIndicator.selectedIndexingSolution = selectedSolution
def process(self, _edObject=None):
EDPluginExec.process(self)
EDVerbose.DEBUG("EDPluginExecReadImageHeaderMARCCDv10.process")
xsDataInputReadImageHeader = self.getDataInput()
xsDataFile = xsDataInputReadImageHeader.getImage()
strPath = xsDataFile.getPath().getValue()
dictMARCCDHeader = self.readHeaderMarccd(strPath)
if (dictMARCCDHeader is None):
strErrorMessage = "EDPluginExecReadImageHeaderMARCCDv10.process : Cannot read header : %s" % strPath
EDVerbose.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
else:
xsDataExperimentalCondition = XSDataExperimentalCondition()
xsDataDetector = XSDataDetector()
iNoPixelsX = int(dictMARCCDHeader["nslow"])
iNoPixelsY = int(dictMARCCDHeader["nfast"])
xsDataDetector.setNumberPixelX(XSDataInteger(iNoPixelsX))
xsDataDetector.setNumberPixelY(XSDataInteger(iNoPixelsY))
fPixelSizeX = float(dictMARCCDHeader["pixelsize_x"]) / 1000.0
xsDataDetector.setPixelSizeX(XSDataLength(fPixelSizeX))
fPixelSizeY = float(dictMARCCDHeader["pixelsize_y"]) / 1000.0
xsDataDetector.setPixelSizeY(XSDataLength(fPixelSizeY))
fBeamPositionX = float(dictMARCCDHeader["beam_x"]) / 1000.0
fBeamPositionY = float(dictMARCCDHeader["beam_y"]) / 1000.0
# Fix for bug 397 - check if the beam position is close to the centre of the image
fTwoTheta = float(dictMARCCDHeader["end_twotheta"]) / 1000.0
xsDataDetector.setTwoTheta(XSDataAngle(fTwoTheta))
if (abs(fTwoTheta) < 0.1):
if (abs(fBeamPositionX /
(fPixelSizeX / 1000.0) - iNoPixelsX / 2.0) >
(2 * iNoPixelsX)):
fBeamPositionX = fBeamPositionX * fPixelSizeX / 1000.0
fBeamPositionY = fBeamPositionY * fPixelSizeY / 1000.0
xsDataDetector.setBeamPositionX(XSDataLength(fBeamPositionX))
xsDataDetector.setBeamPositionY(XSDataLength(fBeamPositionY))
fDistance = float(dictMARCCDHeader["xtal_to_detector"]) / 1000.0
if (abs(fDistance) < 0.1):
fDistanceStart = float(
dictMARCCDHeader["start_xtal_to_detector"]) / 1000.0
fDistanceEnd = float(
dictMARCCDHeader["end_xtal_to_detector"]) / 1000.0
if (abs(fDistanceStart - fDistanceEnd) < 0.1):
fDistance = fDistanceStart
else:
# Somethings very wrong with the distances...
strErrorMessage = "EDPluginExecReadImageHeaderMARCCDv10.process : Inconsistency in MAR CCD image header: start_xtal_to_detector = %d, end_xtal_to_detector = %d" % \
(fDistanceStart, fDistanceEnd)
EDVerbose.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
xsDataDetector.setDistance(XSDataLength(fDistance))
xsDataDetector.setNumberBytesInHeader(
XSDataInteger(float(dictMARCCDHeader["header_size"])))
#xsDataDetector.setSerialNumber( XSDataInteger( dictMARCCDHeader[ "DETECTOR_SN" ] ) ) )
#xsDataDetector.setBin( XSDataString( dictMARCCDHeader[ "BIN" ] ) ) )
#xsDataDetector.setDataType( XSDataString( dictMARCCDHeader[ "TYPE" ] ) ) )
#xsDataDetector.setByteOrder( XSDataString( dictMARCCDHeader[ "BYTE_ORDER" ] ) ) )
xsDataDetector.setImageSaturation(
XSDataInteger(int(dictMARCCDHeader["saturation_level"])))
# Determine type of detector...
if (iNoPixelsX == 2048 and iNoPixelsY == 2048):
xsDataDetector.setName(XSDataString("MAR CCD 165"))
xsDataDetector.setType(XSDataString("mar165"))
elif (iNoPixelsX == 3072 and iNoPixelsY == 3072):
xsDataDetector.setName(XSDataString("MAR CCD 225"))
xsDataDetector.setType(XSDataString("mar225"))
elif (iNoPixelsX == 4096 and iNoPixelsY == 4096):
xsDataDetector.setName(XSDataString("MAR CCD 325"))
xsDataDetector.setType(XSDataString("mar325"))
else:
strErrorMessage = EDMessage.ERROR_DATA_HANDLER_02 % (
"EDPluginExecReadImageHeaderMARCCDv10.process",
"Unknown detector type")
EDVerbose.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError, strErrorMessage
xsDataExperimentalCondition.setDetector(xsDataDetector)
# Beam object
xsDataBeam = XSDataBeam()
xsDataBeam.setWavelength(
XSDataWavelength(
float(dictMARCCDHeader["source_wavelength"]) / 100000.0))
xsDataBeam.setExposureTime(
XSDataTime(float(dictMARCCDHeader["exposure_time"]) / 1000.0))
xsDataExperimentalCondition.setBeam(xsDataBeam)
# Goniostat object
xsDataGoniostat = XSDataGoniostat()
fRotationAxisStart = float(dictMARCCDHeader["start_phi"]) / 1000.0
fOscillationWidth = float(
dictMARCCDHeader["rotation_range"]) / 1000.0
xsDataGoniostat.setRotationAxisStart(
XSDataAngle(fRotationAxisStart))
xsDataGoniostat.setRotationAxisEnd(
XSDataAngle(fRotationAxisStart + fOscillationWidth))
xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth))
xsDataExperimentalCondition.setGoniostat(xsDataGoniostat)
# Create the image object
xsDataImage = XSDataImage()
xsDataImage.setPath(XSDataString(strPath))
strTimeStamp = dictMARCCDHeader["acquire_timestamp"]
xsDataImage.setDate(XSDataString(strTimeStamp))
iImageNumber = EDUtilsImage.getImageNumber(strPath)
xsDataImage.setNumber(XSDataInteger(iImageNumber))
xsDataSubWedge = XSDataSubWedge()
xsDataSubWedge.setExperimentalCondition(
xsDataExperimentalCondition)
xsDataSubWedge.addImage(xsDataImage)
self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader()
self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
EDVerbose.screen("Usage: id29_create_thumbnail image_directory_path image1 [image2]" )
sys.exit(1)
EDVerbose.screen("Arguments: %r" % sys.argv)
# Remove duplicates
listImageName = list(set(sys.argv[2:]))
# Strip off prefix
listPrefix = listImageName[0].split("_")
strPrefix = "{0}_{1}".format(listPrefix[0], listPrefix[1])
# Check if a temp working directory should be created
if "CREATE_THUMBNAIL_WORKING_DIR" in os.environ:
strPathToTempDir = tempfile.mkdtemp(prefix="thumbnail-{0}-".format(strPrefix),
dir=os.environ["CREATE_THUMBNAIL_WORKING_DIR"])
else:
strTmpUser = os.path.join("/tmp", os.environ["USER"])
if not os.path.exists(strTmpUser):
os.mkdir(strTmpUser, 0o755)
strPathToTempDir = tempfile.mkdtemp(prefix="thumbnail-{0}-".format(strPrefix), dir=strTmpUser)
os.chmod(strPathToTempDir, 0o755)
os.chdir(strPathToTempDir)
EDVerbose.setLogFileName(os.path.join(strPathToTempDir, "id29_create_thumbnail.log"))
strImageDirectory = sys.argv[1]
for strImageName in listImageName:
xsDataInputPyarchThumbnailGenerator = XSDataInputPyarchThumbnailGenerator()
xsDataInputPyarchThumbnailGenerator.setWaitForFileTimeOut(XSDataTime(1000))
strImagePath = os.path.join(strImageDirectory, strImageName)
xsDataInputPyarchThumbnailGenerator.setDiffractionImage(XSDataFile(XSDataString(strImagePath)))
EDVerbose.screen("XML input for EDPluginControlPyarchThumbnailGeneratorv1_0: %s" % xsDataInputPyarchThumbnailGenerator.marshal())
edPluginControlPyarchThumbnailGeneratorv1_0 = EDPluginControlPyarchThumbnailGeneratorv1_0()
edPluginControlPyarchThumbnailGeneratorv1_0.setDataInput(xsDataInputPyarchThumbnailGenerator)
edPluginControlPyarchThumbnailGeneratorv1_0.execute()
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG('EDPluginControlAutoproc.preProcess starting')
self.DEBUG('failure state is currently {0}'.format(self.isFailure()))
# for info to send to the autoproc stats server
self.custom_stats = dict(creation_time=time.time(),
processing_type='edna fastproc',
datacollect_id=self.dataInput.data_collection_id.value,
comments='running on {0}'.format(socket.gethostname()))
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
xds_in.spacegroup = data_in.spacegroup
xds_in.unit_cell = data_in.unit_cell
self.log_file_path = os.path.join(self.root_dir, 'stats.json')
self.DEBUG('will log timing information to {0}'.format(self.log_file_path))
self.stats = dict()
# Get the image prefix from the directory name
# XXX: This is horrible
try:
self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1])
except Exception:
self.image_prefix = ''
self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing')
try:
os.makedirs(self.results_dir)
except OSError: # it most likely exists
pass
# Copy the vanilla XDS input file to the results dir
infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP')
shutil.copy(self.dataInput.input_file.path.value,
infile_dest)
# Ensure the autoproc ids directory is there
self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids')
try:
os.makedirs(self.autoproc_ids_dir)
except OSError: # it's there
pass
# we'll need the low res limit later on
lowres = data_in.low_resolution_limit
if lowres is not None:
self.low_resolution_limit = lowres.value
else:
self.low_resolution_limit = 50
res_override = data_in.res_override
if res_override is not None:
self.res_override = res_override.value
else:
# XXX: default to 0?
self.res_override = None
# check the number of images (must be > 8) and get the first
# image name to wait for. Also modify the XDS.INP file to
# reflect these values, if specified
conf = parse_xds_file(data_in.input_file.path.value)
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in conf:
xgeo = os.path.abspath(os.path.join(self.root_dir,
conf['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
del conf['X-GEO_CORR=']
else:
conf['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in conf:
ygeo = os.path.abspath(os.path.join(self.root_dir,
conf['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
del conf['Y-GEO_CORR=']
else:
conf['Y-GEO_CORR='] = ygeo
dump_xds_file(data_in.input_file.path.value, conf)
resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')
if resrange is not None:
if self.low_resolution_limit is not None:
resrange[0] = self.low_resolution_limit
if self.res_override is not None:
resrange[1] = self.res_override
conf['INCLUDE_RESOLUTION_RANGE='] = resrange
dump_xds_file(data_in.input_file.path.value, conf)
data_range = conf.get('DATA_RANGE=')
# we'll need that for the very last part ( file import )
self.data_range = data_range
if data_range is not None:
start_image = data_range[0]
end_image = data_range[1]
if end_image - start_image < 8:
self.ERROR('there are fewer than 8 images, aborting')
self.setFailure()
return
template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
self.DEBUG('template for images is {0}'.format(template))
# fix the path if it's not absolute
if not os.path.isabs(template):
self.DEBUG('file template {0} is not absolute'.format(template))
base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value))
template = os.path.normpath(os.path.join(self.root_dir, template))
conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template
self.DEBUG('file template fixed to {0}'.format(template))
self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value))
dump_xds_file(data_in.input_file.path.value, conf)
first_image = _template_to_image(template, start_image)
self.wait_file = self.loadPlugin('EDPluginWaitFile')
waitfileinput = XSDataInputWaitFile()
waitfileinput.expectedFile = XSDataFile()
waitfileinput.expectedFile.path = XSDataString(first_image)
waitfileinput.expectedSize = XSDataInteger(0) # we do not care
timeout = XSDataTime()
global WAIT_FOR_FRAME_TIMEOUT
timeout.value = WAIT_FOR_FRAME_TIMEOUT
waitfileinput.timeOut = timeout
self.wait_file.dataInput = waitfileinput
self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")
self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')
self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def testSetDataModelInput(self):
# Crystal
from XSDataRaddosev10 import XSDataRaddoseInput
xsDataRaddoseInput = XSDataRaddoseInput()
from XSDataCommon import XSDataString
from XSDataCommon import XSDataFloat
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataLength
from XSDataCommon import XSDataSize
from XSDataCommon import XSDataInteger
from XSDataRaddosev10 import XSDataCell
from XSDataRaddosev10 import XSDataAtom
from XSDataRaddosev10 import XSDataAtomicComposition
xsDataAtomSulfur = XSDataAtom()
xsDataAtomSulfur.setNumberOf(XSDataFloat(4))
xsDataAtomSulfur.setSymbol(XSDataString("S"))
xsDataAtomSelenium = XSDataAtom()
xsDataAtomSelenium.setNumberOf(XSDataFloat(4))
xsDataAtomSelenium.setSymbol(XSDataString("Se"))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtomicComposition.addAtom(xsDataAtomSulfur)
xsDataAtomicComposition.addAtom(xsDataAtomSelenium)
xsDataRaddoseInput.setCrystalPATM(xsDataAtomicComposition)
xsDataRaddoseInput.setCrystalNRES(XSDataInteger(295))
xsDataRaddoseInput.setCrystalNMON(XSDataInteger(8))
xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0),
angle_beta=XSDataAngle(90.0),
angle_gamma=XSDataAngle(90.0),
length_a=XSDataLength(78.9),
length_b=XSDataLength(95.162),
length_c=XSDataLength(104.087))
xsDataSizeCrystal = XSDataSize(XSDataLength(0.1), XSDataLength(0.1),
XSDataLength(0.1))
xsDataRaddoseInput.setCrystalCell(xsDataCell)
xsDataRaddoseInput.setCrystalSize(xsDataSizeCrystal)
# Beam
from XSDataCommon import XSDataFlux
from XSDataCommon import XSDataWavelength
from XSDataCommon import XSDataTime
xsDataSize = XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1))
xsDataRaddoseInput.setBeamSize(xsDataSize)
xsDataRaddoseInput.setBeamFlux(XSDataFlux(1e+12))
xsDataRaddoseInput.setBeamWavelength(XSDataWavelength(2.41))
xsDataRaddoseInput.setBeamExposureTime(XSDataTime(1))
xsDataRaddoseInput.setNumberOfImages(XSDataInteger(1))
xsDataRaddoseInput.exportToFile(self.strObtainedInputFile)
strExpectedInput = self.readAndParseFile(self.strReferenceInputFile)
strObtainedInput = self.readAndParseFile(self.strObtainedInputFile)
xsDataRaddoseInputExpected = XSDataRaddoseInput.parseString(
strExpectedInput)
xsDataRaddoseInputObtained = XSDataRaddoseInput.parseString(
strObtainedInput)
EDAssert.equal(xsDataRaddoseInputExpected.marshal(),
xsDataRaddoseInputObtained.marshal())
def postProcess(self, _edObject=None):
EDPluginExecProcessScript.postProcess(self)
strRaddoseError = self.readProcessErrorLogFile()
if ((strRaddoseError is not None) and (strRaddoseError != "")):
errorMessage = EDMessage.ERROR_EXECUTION_03 % (
'EDPluginRaddosev10.postProcess', 'EDPluginRaddosev10',
strRaddoseError)
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError(errorMessage)
resultDict = self.analyseScriptLogFileName(
os.path.join(self.getWorkingDirectory(),
self.getScriptLogFileName()))
strSolvent = self.__dictResults[EDPluginRaddosev10.__strSOLVENT]
# ugly workaround while waiting for RADDOSE XML output file
strAbsorbedDose = None
for strAbsorbedDoseKeyword in EDPluginRaddosev10.__listABSORBED_DOSE:
strAbsorbedDose = self.__dictResults[strAbsorbedDoseKeyword]
if (strAbsorbedDose is not None):
break
if (strSolvent is None):
errorMessage = EDMessage.ERROR_EXECUTION_03 % (
'EDPluginRaddosev10.postProcess', "Raddose",
"No Result for Keyword [" + EDPluginRaddosev10.__strSOLVENT +
"] see: " + self.getScriptLogFileName())
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError(errorMessage)
if (strAbsorbedDose is None):
errorMessage = EDMessage.ERROR_EXECUTION_03 % (
'EDPluginRaddosev10.postProcess', "Raddose",
"No Result for Keyword [" + strAbsorbedDoseKeyword +
"] see: " + self.getScriptLogFileName())
self.error(errorMessage)
self.addErrorMessage(errorMessage)
raise RuntimeError(errorMessage)
self.__fSolvent = float(strSolvent)
if (self.__fSolvent < EDPluginRaddosev10.__iMIN_SOLVENT_PERCENTAGE
or self.__fSolvent >
EDPluginRaddosev10.__iMAX_SOLVENT_PERCENTAGE):
warningMessage = "Inconsistent solvent percentage value: %.1f" % self.__fSolvent
self.warning(warningMessage)
self.addWarningMessage(warningMessage)
iNumberOfImages = self.getDataInput().getNumberOfImages().getValue()
fExposureTimePerImageInSec = self.getDataInput().getBeamExposureTime(
).getValue()
fTotalExposureTime = iNumberOfImages * fExposureTimePerImageInSec
fAbsorbedDoseSpeed = float(strAbsorbedDose) / fTotalExposureTime
xsDataAbsorbedDoseRate = XSDataAbsorbedDoseRate(fAbsorbedDoseSpeed)
xsDataAbsorbedDose = XSDataDouble(float(strAbsorbedDose))
self.__fTimeToReachHendersonLimit = EDPluginRaddosev10.__fHENDERSON_LIMIT / fAbsorbedDoseSpeed
xsDataRaddosev10Output = XSDataRaddoseOutput()
xsDataRaddosev10Output.setAbsorbedDose(xsDataAbsorbedDose)
xsDataRaddosev10Output.setAbsorbedDoseRate(xsDataAbsorbedDoseRate)
xsDataRaddosev10Output.setTimeToReachHendersonLimit(
XSDataTime(self.__fTimeToReachHendersonLimit))
xsDataFilePathToLog = XSDataFile(
XSDataString(
os.path.join(self.getWorkingDirectory(),
self.getScriptLogFileName())))
xsDataRaddosev10Output.setPathToLogFile(xsDataFilePathToLog)
self.setDataOutput(xsDataRaddosev10Output)
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG('EDPluginControlAutoproc.preProcess starting')
self.DEBUG('failure state is currently {0}'.format(self.isFailure()))
# for info to send to the autoproc stats server
self.custom_stats = dict(
creation_time=time.time(),
processing_type='edna fastproc',
datacollect_id=self.dataInput.data_collection_id.value,
comments='running on {0}'.format(socket.gethostname()))
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
xds_in.spacegroup = data_in.spacegroup
xds_in.unit_cell = data_in.unit_cell
self.log_file_path = os.path.join(self.root_dir, 'stats.json')
self.DEBUG('will log timing information to {0}'.format(
self.log_file_path))
self.stats = dict()
# Get the image prefix from the directory name
# XXX: This is horrible
try:
self.image_prefix = '_'.join(
os.path.basename(self.root_dir).split('_')[1:-1])
except Exception:
self.image_prefix = ''
self.results_dir = os.path.join(self.root_dir, 'results',
'fast_processing')
try:
os.makedirs(self.results_dir)
except OSError: # it most likely exists
pass
# Copy the vanilla XDS input file to the results dir
infile_dest = os.path.join(self.results_dir,
self.image_prefix + '_input_XDS.INP')
shutil.copy(self.dataInput.input_file.path.value, infile_dest)
# Ensure the autoproc ids directory is there
self.autoproc_ids_dir = os.path.join(self.results_dir,
'fastproc_integration_ids')
try:
os.makedirs(self.autoproc_ids_dir)
except OSError: # it's there
pass
# we'll need the low res limit later on
lowres = data_in.low_resolution_limit
if lowres is not None:
self.low_resolution_limit = lowres.value
else:
self.low_resolution_limit = 50
res_override = data_in.res_override
if res_override is not None:
self.res_override = res_override.value
else:
# XXX: default to 0?
self.res_override = None
# check the number of images (must be > 8) and get the first
# image name to wait for. Also modify the XDS.INP file to
# reflect these values, if specified
conf = parse_xds_file(data_in.input_file.path.value)
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in conf:
xgeo = os.path.abspath(
os.path.join(self.root_dir, conf['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(xgeo))
del conf['X-GEO_CORR=']
else:
conf['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in conf:
ygeo = os.path.abspath(
os.path.join(self.root_dir, conf['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(ygeo))
del conf['Y-GEO_CORR=']
else:
conf['Y-GEO_CORR='] = ygeo
dump_xds_file(data_in.input_file.path.value, conf)
resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')
if resrange is not None:
if self.low_resolution_limit is not None:
resrange[0] = self.low_resolution_limit
if self.res_override is not None:
resrange[1] = self.res_override
conf['INCLUDE_RESOLUTION_RANGE='] = resrange
dump_xds_file(data_in.input_file.path.value, conf)
data_range = conf.get('DATA_RANGE=')
# we'll need that for the very last part ( file import )
self.data_range = data_range
if data_range is not None:
start_image = data_range[0]
end_image = data_range[1]
if end_image - start_image < 8:
self.ERROR('there are fewer than 8 images, aborting')
self.setFailure()
return
template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
self.DEBUG('template for images is {0}'.format(template))
# fix the path if it's not absolute
if not os.path.isabs(template):
self.DEBUG('file template {0} is not absolute'.format(template))
base_dir = os.path.abspath(
os.path.dirname(data_in.input_file.path.value))
template = os.path.normpath(os.path.join(self.root_dir, template))
conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template
self.DEBUG('file template fixed to {0}'.format(template))
self.DEBUG('dumping back the file to {0}'.format(
data_in.input_file.path.value))
dump_xds_file(data_in.input_file.path.value, conf)
first_image = _template_to_image(template, start_image)
self.wait_file = self.loadPlugin('EDPluginWaitFile')
waitfileinput = XSDataInputWaitFile()
waitfileinput.expectedFile = XSDataFile()
waitfileinput.expectedFile.path = XSDataString(first_image)
waitfileinput.expectedSize = XSDataInteger(0) # we do not care
timeout = XSDataTime()
global WAIT_FOR_FRAME_TIMEOUT
timeout.value = WAIT_FOR_FRAME_TIMEOUT
waitfileinput.timeOut = timeout
self.wait_file.dataInput = waitfileinput
self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")
self.store_autoproc_anom = self.loadPlugin(
'EDPluginISPyBStoreAutoProcv1_4')
self.store_autoproc_noanom = self.loadPlugin(
'EDPluginISPyBStoreAutoProcv1_4')
self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')
self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def buildChildren(self, child_, nodeName_): if child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamExposureTime': obj_ = XSDataTime() obj_.build(child_) self.setBeamExposureTime(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamFlux': obj_ = XSDataFlux() obj_.build(child_) self.setBeamFlux(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamSize': obj_ = XSDataSize() obj_.build(child_) self.setBeamSize(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'beamWavelength': obj_ = XSDataWavelength() obj_.build(child_) self.setBeamWavelength(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalCell': obj_ = XSDataCell() obj_.build(child_) self.setCrystalCell(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNDNA': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNDNA(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNMON': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNMON(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNRES': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNRES(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalNRNA': obj_ = XSDataInteger() obj_.build(child_) self.setCrystalNRNA(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalPATM': obj_ = XSDataAtomicComposition() obj_.build(child_) self.setCrystalPATM(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalSATM': obj_ = XSDataAtomicComposition() obj_.build(child_) self.setCrystalSATM(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'crystalSize': obj_ = XSDataSize() obj_.build(child_) self.setCrystalSize(obj_) elif child_.nodeType == Node.ELEMENT_NODE and \ nodeName_ == 'numberOfImages': obj_ = XSDataInteger() obj_.build(child_) self.setNumberOfImages(obj_) XSDataInput.buildChildren(self, child_, nodeName_)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
if self.dataInput.startImageNumber is not None:
imageNoStart = self.dataInput.startImageNumber.value
if self.dataInput.endImageNumber is not None:
imageNoEnd = self.dataInput.endImageNumber.value
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if imageNoStart is None:
imageNoStart = ispybDataCollection.startImageNumber
if imageNoEnd is None:
imageNoEnd = ispybDataCollection.startImageNumber + \
ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory,
fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XDSAPP", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(
self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
if self.dataInput.hdf5ToCbfDirectory is not None:
dir = self.dataInput.hdf5ToCbfDirectory.path.value
pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]
self.timeStart = time.localtime()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
xsDataInputXDSAPPAnom.image = XSDataFile(
XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPAnom.spacegroup = XSDataString(
self.xdsAppSpacegroup)
self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
self.edPluginExecXDSAPPAnom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
xsDataInputXDSAPPNoanom.image = XSDataFile(
XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPNoanom.spacegroup = XSDataString(
self.xdsAppSpacegroup)
self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
self.edPluginExecXDSAPPNoanom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXDSAPPAnom.synchronize()
xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
# Run XSCALE even if XSCALE.LP is present
strPathXscaleLpAnom = self.runXscale(
self.edPluginExecXDSAPPAnom.getWorkingDirectory(),
anom=True,
merged=True)
if self.doNoanom:
self.edPluginExecXDSAPPNoanom.synchronize()
xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
strPathXscaleLpNoanom = self.runXscale(
self.edPluginExecXDSAPPNoanom.getWorkingDirectory(),
anom=False,
merged=True)
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.dataInput.dataCollectionId is not None:
# Check if we should use XDS_ASCII_to_XML.pl
if self.doAnom:
if self.useXdsAsciiToXml:
# Only for anom runs
self.runXdsAsciiToXml(
xsDataResultXDSAPPAnom,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom,
self.autoProcProgramIdAnom)
else:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
xsDataResultXDSAPPAnom, processDirectory, template,
strPathXscaleLpAnom, True, proposal, self.timeStart,
self.timeEnd, self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom,
self.autoProcProgramIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
xsDataResultXDSAPPNoanom, processDirectory, template,
strPathXscaleLpNoanom, False, proposal, self.timeStart,
self.timeEnd, self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdNoanom,
self.autoProcProgramIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlDozorv1_1.process")
msg = "EDPluginControlDozorXmlRpcv1_0 started..."
self.sendMessageToMXCuBE("Processing started...", "info")
xsDataResultControlDozor = XSDataResultControlDozor()
firstFileName = self.dataInput.template.value % (
self.dataInput.first_run_number.value,
self.dataInput.first_image_number.value)
fWaitFileTimeout = 180 #sec
self.waitFileFirst = self.loadPlugin("EDPluginMXWaitFilev1_1",
"MXWaitFileFirst")
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(firstFileName))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.waitFileFirst.size = XSDataInteger(1000000)
self.waitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.waitFileFirst.executeSynchronous()
if self.waitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, firstFileName)
#self.addWarningMessage(strWarningMessage)
#self.WARNING(strWarningMessage)
#self.sendMessageToMXCuBE(strWarningMessage, "error")
edPluginControlReadImageHeader = self.loadPlugin(
self.strEDPluginControlReadImageHeaderName)
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = XSDataFile(
XSDataString(firstFileName))
edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
edPluginControlReadImageHeader.executeSynchronous()
subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
xsDataInputDozor = XSDataInputDozor()
beam = subWedge.experimentalCondition.beam
detector = subWedge.experimentalCondition.detector
goniostat = subWedge.experimentalCondition.goniostat
xsDataInputDozor.detectorType = detector.type
xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
xsDataInputDozor.spotSize = XSDataDouble(3.0)
xsDataInputDozor.detectorDistance = XSDataDouble(
detector.distance.value)
xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
orgx = detector.beamPositionY.value / detector.pixelSizeY.value
orgy = detector.beamPositionX.value / detector.pixelSizeX.value
xsDataInputDozor.orgx = XSDataDouble(orgx)
xsDataInputDozor.orgy = XSDataDouble(orgy)
# GB: the 50 might need tunig to CPU speed and number of. 2000 is a current limit of Dozor.
self.maxChunkSize = 4000 #a min (150 * 2527 * 2463 /detector.numberPixelX.value / detector.numberPixelY.value, 2000)
_beamstop = self.beamstop(detector)
if _beamstop is not None:
#self.WARNING("Setting beamstop shadow: %s"%_beamstop)
xsDataInputDozor.ixMin = XSDataInteger(_beamstop['ix_min'])
xsDataInputDozor.iyMin = XSDataInteger(_beamstop['iy_min'])
xsDataInputDozor.ixMax = XSDataInteger(_beamstop['ix_max'])
xsDataInputDozor.iyMax = XSDataInteger(_beamstop['iy_max'])
if self.dataInput.pixelMin is not None:
xsDataInputDozor.pixelMin = self.dataInput.pixelMin
if self.dataInput.pixelMax is not None:
xsDataInputDozor.pixelMax = self.dataInput.pixelMax
_serial = 0
_startTime = time.time()
chunk_list = self.schedule(goniostat.rotationAxisStart.value,
goniostat.oscillationWidth.value)
for chunk in chunk_list:
if not self.poll_file(
self.dataInput.template.value %
(chunk['run_number'], chunk['first'] + chunk['number_of'] - 1),
(beam.exposureTime.value + 0.003) * chunk['number_of'] + 30):
self.sendMessageToMXCuBE("Timeout waiting for frame: %d" %
(chunk['first'] + chunk['number_of']),
level="error")
return
xsDataInputDozor.oscillationRange = XSDataDouble(
chunk['rotation_range'])
xsDataInputDozor.startingAngle = XSDataDouble(
chunk['rotation_start'])
xsDataInputDozor.firstImageNumber = XSDataInteger(chunk['first'])
xsDataInputDozor.numberImages = XSDataInteger(chunk['number_of'])
strFileName = self.dataInput.template.value % (chunk['run_number'],
chunk['first'])
strXDSTemplate = EDUtilsImage.getTemplate(strFileName, '?')
xsDataInputDozor.nameTemplateImage = XSDataString(
os.path.join(os.path.dirname(strFileName), strXDSTemplate))
edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor")
edPluginDozor.dataInput = xsDataInputDozor
edPluginDozor.executeSynchronous()
xsDataChunkResultControlImageDozor = XSDataResultControlDozor()
dozor_batch_list = [] #[[1,4,3,8,6],
#[2, 5, 3, 3, 7],[1, 3, 12, 11, 4],[4, 2, 3, 5, 7]]
#ozor_image_dict = {}
diff_image_count = 0
xsDataControlImageDozor = XSDataControlImageDozor()
for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
xsDataControlImageDozor = XSDataControlImageDozor()
xsDataControlImageDozor.number = xsDataResultDozor.number
strFileName = self.dataInput.template.value % (
chunk['run_number'], xsDataControlImageDozor.number.value)
xsDataControlImageDozor.image = XSDataFile(
XSDataString(strFileName))
xsDataControlImageDozor.spots_num_of = xsDataResultDozor.spots_num_of
xsDataControlImageDozor.spots_int_aver = xsDataResultDozor.spots_int_aver
xsDataControlImageDozor.spots_resolution = xsDataResultDozor.spots_resolution
xsDataControlImageDozor.powder_wilson_scale = xsDataResultDozor.powder_wilson_scale
xsDataControlImageDozor.powder_wilson_bfactor = xsDataResultDozor.powder_wilson_bfactor
xsDataControlImageDozor.powder_wilson_resolution = xsDataResultDozor.powder_wilson_resolution
xsDataControlImageDozor.powder_wilson_correlation = xsDataResultDozor.powder_wilson_correlation
xsDataControlImageDozor.powder_wilson_rfactor = xsDataResultDozor.powder_wilson_rfactor
xsDataControlImageDozor.score = xsDataResultDozor.score
xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
dozor_batch_list.append(
(xsDataControlImageDozor.number.getValue(),
xsDataControlImageDozor.spots_num_of.getValue(),
xsDataControlImageDozor.spots_int_aver.getValue(),
xsDataControlImageDozor.spots_resolution.getValue(),
xsDataControlImageDozor.score.getValue()))
if xsDataControlImageDozor.spots_num_of.getValue() > 0:
diff_image_count += 1
xsDataChunkResultControlImageDozor.addImageDozor(
xsDataControlImageDozor)
xsDataChunkResultControlImageDozor.exportToFile(
"ResultControlDozor_Chunk_%06d.xml" % _serial)
self.sendResultToMXCuBE(dozor_batch_list)
_serial += 1
self.screen("Chunk %d/%d done in %.3f seconds" % \
(_serial, len(chunk_list), time.time()-_startTime))
self.sendMessageToMXCuBE("Chunk %d/%d done in %.2f sec., num diffr. frames: %d/%d" % \
(_serial,
len(chunk_list),
time.time() - _startTime,
diff_image_count,
len(dozor_batch_list)
)
)
_startTime = time.time()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_3.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist" % dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist" % inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], self.lstInput[1]])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.lstSummary.append("WARNING: " + self.strRadiationDamage)
self.lstSummary.append("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
#outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer
is not None) and (self.__class__.lastSample
is not None):
self.__edPluginExecAutoSub = self.loadPlugin(
self.__strControlledPluginAutoSub)
base = "_".join(
os.path.basename(
self.__class__.lastSample.path.value).split("_")
[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile),
base + "_" + suff)
xsd = XSDataInputAutoSub(
sampleCurve=self.__class__.lastSample,
buffers=[
self.__class__.lastBuffer,
self.dataInput.mergedCurve
],
subtractedCurve=XSDataFile(XSDataString(sub)))
self.__edPluginExecAutoSub.setDataInput(xsd)
self.__edPluginExecAutoSub.connectSUCCESS(
self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(
self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
self.__class__.lastSample = None
else:
self.__class__.lastSample = self.dataInput.mergedCurve
def getXSDataRaddoseInput(self, _xsDataBeam, _xsDataSample, _inumOperators):
xsDataRaddoseInput = XSDataRaddoseInput()
# Beam
xsDataRaddoseInput.setBeamSize(_xsDataBeam.getSize())
xsDataRaddoseInput.setBeamFlux(_xsDataBeam.getFlux())
xsDataRaddoseInput.setBeamWavelength(_xsDataBeam.getWavelength())
xsDataRaddoseInput.setBeamExposureTime(XSDataTime(1.0))
xsDataRaddoseInput.setNumberOfImages(XSDataInteger(1))
xsDataRaddoseInput.setCrystalCell(_xsDataSample.getCrystal().getCell())
xsDataRaddoseInput.setCrystalSize(_xsDataSample.getSize())
xsDataComposition = _xsDataSample.getChemicalComposition()
xsDataSolvent = xsDataComposition.getSolvent()
if(xsDataSolvent is not None):
xsDataRaddoseInput.setCrystalSATM(xsDataSolvent.getAtoms())
xsDataStructure = xsDataComposition.getStructure()
if(xsDataStructure is not None):
xsDataChains = xsDataStructure.getChain()
totalNRESInStructure = 0
totalNDNAInStructure = 0
totalNRNAInStructure = 0
totalPATM = XSDataAtomicComposition()
for chain in xsDataChains:
# heavy atoms of each chain to be added in the PATM
xsDataAtomicCompositionHeavyAtoms = chain.getHeavyAtoms()
if (xsDataAtomicCompositionHeavyAtoms is not None):
iterator = 1
while iterator <= chain.getNumberOfCopies().getValue():
totalPATM = self.mergeAtomicComposition(totalPATM, xsDataAtomicCompositionHeavyAtoms)
iterator = iterator + 1
type = chain.getType().getValue()
numberOfMonomers = chain.getNumberOfMonomers().getValue() * chain.getNumberOfCopies().getValue()
if (type == "protein"):
totalNRESInStructure = totalNRESInStructure + numberOfMonomers
elif(type == "dna"):
totalNDNAInStructure = totalNDNAInStructure + numberOfMonomers
elif(type == "rna"):
totalNRNAInStructure = totalNRNAInStructure + numberOfMonomers
xsDataLigands = xsDataStructure.getLigand()
for ligand in xsDataLigands:
# Light atoms to be added to the NRES
nres = ligand.getNumberOfLightAtoms().getValue() * ligand.getNumberOfCopies().getValue() / 7.85
totalNRESInStructure = totalNRESInStructure + nres
# Heavy atoms to be added to the PATM
if (ligand.getHeavyAtoms() is not None):
iterator = 1
while iterator <= ligand.getNumberOfCopies().getValue():
totalPATM = self.mergeAtomicComposition(totalPATM, ligand.getHeavyAtoms())
iterator = iterator + 1
if(totalNRESInStructure != 0):
xsDataRaddoseInput.setCrystalNRES(XSDataInteger(int(round(totalNRESInStructure))))
if(totalNDNAInStructure != 0):
xsDataRaddoseInput.setCrystalNDNA(XSDataInteger(int(totalNDNAInStructure)))
if(totalNRNAInStructure != 0):
xsDataRaddoseInput.setCrystalNRNA(XSDataInteger(int(totalNRNAInStructure)))
if(len(totalPATM.getAtom()) != 0):
xsDataRaddoseInput.setCrystalPATM(totalPATM)
xsDataNumberNumStructInAU = xsDataStructure.getNumberOfCopiesInAsymmetricUnit()
xsDataNumberNumStructInUC = int(xsDataNumberNumStructInAU.getValue() * _inumOperators)
xsDataRaddoseInput.setCrystalNMON(XSDataInteger(xsDataNumberNumStructInUC))
return xsDataRaddoseInput
def testIsSameExperimentalCondition(self):
edPluginSubWedgeMergev1_1 = self.createPlugin()
xsDataExperimentalConditionReference = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionSameAsReference = self.getTestExperimentalCondition(
)
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionSameAsReference), True)
xsDataExperimentalConditionDifferentExposureTime = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentExposureTime.getBeam(
).setExposureTime(XSDataTime(10.0))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentExposureTime), False)
xsDataExperimentalConditionDifferentWavelength = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentWavelength.getBeam().setWavelength(
XSDataWavelength(1.5))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentWavelength), False)
xsDataExperimentalConditionDifferentBeamPositionX = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentBeamPositionX.getDetector(
).setBeamPositionX(XSDataLength(20.0))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentBeamPositionX), False)
xsDataExperimentalConditionDifferentBeamPositionY = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentBeamPositionY.getDetector(
).setBeamPositionY(XSDataLength(20.0))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentBeamPositionY), False)
xsDataExperimentalConditionDifferentDistance = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentDistance.getDetector().setDistance(
XSDataLength(220.0))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentDistance), False)
xsDataExperimentalConditionDifferentName = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentName.getDetector().setName(
XSDataString(u"EDNA"))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentName), False)
xsDataExperimentalConditionDifferentNumberPixelX = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentNumberPixelX.getDetector(
).setNumberPixelX(XSDataInteger(2))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentNumberPixelX), False)
xsDataExperimentalConditionDifferentNumberPixelY = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentNumberPixelY.getDetector(
).setNumberPixelY(XSDataInteger(2))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentNumberPixelY), False)
xsDataExperimentalConditionDifferentSerialNumber = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentSerialNumber.getDetector(
).setSerialNumber(XSDataString(u"EDNA"))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentSerialNumber), False)
xsDataExperimentalConditionDifferentTwoTheta = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentTwoTheta.getDetector().setTwoTheta(
XSDataAngle(90.0))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentTwoTheta), False)
xsDataExperimentalConditionDifferentOscillationWidth = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentOscillationWidth.getGoniostat(
).setOscillationWidth(XSDataAngle(2.0))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentOscillationWidth), False)
xsDataExperimentalConditionDifferentRotationAxis = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentRotationAxis.getGoniostat(
).setRotationAxis(XSDataString(u"EDNA"))
EDAssert.equal(
edPluginSubWedgeMergev1_1.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentRotationAxis), False)
def createInputCharacterisationFromSubWedges(self):
EDVerbose.DEBUG(
"EDPluginControlInterfacev1_2.createInputCharacterisationFromSubWedges"
)
xsDataResultSubWedgeAssemble = self.edPluginControlSubWedgeAssemble.getDataOutput(
)
self.xsDataInputCharacterisation = XSDataInputCharacterisation()
xsDataCollection = XSDataCollection()
# Default exposure time (for the moment, this value should be
# possible to read from the command line)
if self.xsDataDiffractionPlan is None:
self.xsDataDiffractionPlan = XSDataDiffractionPlan()
if (not xsDataResultSubWedgeAssemble is None):
pyListSubWedge = xsDataResultSubWedgeAssemble.getSubWedge()
xsDataCollection.setSubWedge(pyListSubWedge)
for xsDataSubWedge in pyListSubWedge:
if (self.strComplexity is not None):
self.xsDataDiffractionPlan.setComplexity(
XSDataString(self.strComplexity))
if (self.fFlux is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setFlux(XSDataFlux(self.fFlux))
if (self.fBeamSizeX is not None) and (self.fBeamSizeY
is not None):
xsDataSize = XSDataSize()
xsDataSize.setX(XSDataLength(self.fBeamSizeX))
xsDataSize.setY(XSDataLength(self.fBeamSizeY))
xsDataSubWedge.getExperimentalCondition().getBeam(
).setSize(xsDataSize)
if (self.fBeamPosX is not None):
xsDataSubWedge.getExperimentalCondition().getDetector(
).setBeamPositionX(XSDataLength(self.fBeamPosX))
if (self.fBeamPosY is not None):
xsDataSubWedge.getExperimentalCondition().getDetector(
).setBeamPositionY(XSDataLength(self.fBeamPosY))
if (self.fMinExposureTimePerImage is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setMinExposureTimePerImage(
XSDataTime(self.fMinExposureTimePerImage))
if (self.fTransmission is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setTransmission(XSDataDouble(self.fTransmission))
if (self.fWavelength is not None):
xsDataSubWedge.getExperimentalCondition().getBeam(
).setWavelength(XSDataWavelength(self.fWavelength))
if self.fMinOscillationWidth != None:
xsDataSubWedge.getExperimentalCondition().getGoniostat(
).setMinOscillationWidth(
XSDataAngle(self.fMinOscillationWidth))
if self.fMaxOscillationSpeed != None:
xsDataSubWedge.getExperimentalCondition().getGoniostat(
).setMaxOscillationSpeed(
XSDataAngularSpeed(self.fMaxOscillationSpeed))
if (self.strForcedSpaceGroup is not None):
self.xsDataDiffractionPlan.setForcedSpaceGroup(
XSDataString(self.strForcedSpaceGroup))
self.xsDataDiffractionPlan.setAnomalousData(
XSDataBoolean(self.bAnomalousData))
self.xsDataDiffractionPlan.setMaxExposureTimePerDataCollection(
XSDataTime(self.fMaxExposureTimePerDataCollection))
if (self.strStrategyOption is not None):
self.xsDataDiffractionPlan.setStrategyOption(
XSDataString(self.strStrategyOption))
xsDataCollection.setDiffractionPlan(self.xsDataDiffractionPlan)
if self.xsDataSample is not None:
xsDataCollection.setSample(
XSDataSampleCrystalMM.parseString(self.xsDataSample.marshal()))
self.xsDataInputCharacterisation.setDataCollection(xsDataCollection)
def makeXML(self, filename):
"""Here we create the XML string to be passed to the EDNA plugin from the input filename
This can / should be modified by the final user
@param filename: full path of the input file
@type filename: python string representing the path
@rtype: XML string
@return: python string
"""
self.header = None
dirname, basename = os.path.split(filename)
if not basename.startswith(self.prefix):
return
if self.normalizedSuffix and basename.endswith(self.normalizedSuffix):
return
if basename.startswith(self.flatPrefix):
return
if not basename.endswith(self.suffix):
return
if self.fScaleData:
fScaleData = self.fScaleData
else:
fScaleData = self.getNbFrames(filename)
if not fScaleData:
fScaleData = 1.0
if self.fScaleDark:
fScaleDark = self.fScaleDark
else:
fScaleDark = self.getNbFrames(self.darks[0]["path"])
if not fScaleDark:
fScaleDark = 1.0
if isinstance(self.reference, int):
reference = self.reference
else:
reference = -1
xsd = XSDataInputFullFieldXAS(HDF5File=XSDataFile(path=XSDataString(self.HDF5)),
internalHDF5Path=XSDataString(self.internalHdf5),
measureOffset=self.getXSDMeasureOffset(),
dark=self.getXsdDark(),
reference=XSDataInteger(reference),
data=[XSDataImageExt(path=XSDataString(filename),
exposureTime=XSDataTime(self.getExposureTime(filename)))],
dataScaleFactor=XSDataDouble(fScaleData),
darkScaleFactor=XSDataDouble(fScaleDark),
)
if self.dontAlign:
xsd.dontAlign = XSDataBoolean(self.dontAlign)
extendedPrefix = ""
number = ""
started = False
if self.lstSubscanSize:
subScanDigit = []
for i in basename[len(self.prefix):]:
if started and i == "_":
if len(number) > 0:
subScanDigit.append(number)
number = ""
continue
if started and not i.isdigit():
if number:
subScanDigit.append(number)
number = ""
break
if not started:
if i.isdigit():
started = True
else:
extendedPrefix += i
if started:
number += i
if not subScanDigit:
print("ERROR: no index guessed !!!")
return ""
elif len(subScanDigit) == 1:
index = int(subScanDigit[0])
else:# len(subScanDigit) > 1:
index = 0
for i in range(int(subScanDigit[0])):
index += self.lstSubscanSize[i]
index += int(subScanDigit[1])
extendedPrefix += "_".join(subScanDigit[:2])
else:
for i in basename[len(self.prefix):]:
extendedPrefix += i
if started and not i.isdigit():
break
if not started and i.isdigit():
started = True
if started:
number += i
index = int(number)
xsd.index = XSDataInteger(index)
if self.normalizedSuffix:
pr = os.path.splitext(os.path.abspath(filename))[0]
xsd.saveNormalized = XSDataFile(path=XSDataString(pr + self.normalizedSuffix))
energy = self.getEnergy(filename)
if energy is not None:
xsd.energy = XSDataDouble(energy)
flatprefix = self.flatPrefix + extendedPrefix
listFlats = []
for oneFile in os.listdir(dirname):
if oneFile.startswith(flatprefix) and oneFile.endswith(self.suffix):
oneCompleteFile = os.path.abspath(os.path.join(dirname, oneFile))
xsdFileFlat1 = XSDataImageExt(path=XSDataString(oneCompleteFile),
exposureTime=XSDataTime(self.getExposureTime(oneCompleteFile)))
listFlats.append(xsdFileFlat1)
xsd.flat = listFlats
if len(listFlats) != 2:
EDVerbose.WARNING("Not exactly 2 flats were found:" + " ".join([a.path.value for a in listFlats ]))
EDVerbose.WARNING("Those are the flat prefix ans suffix: %s\t%s" % (flatprefix, self.suffix))
if self.fScaleFlat:
fScaleFlat = self.fScaleFlat
else:
fScaleFlat = self.getNbFrames(oneCompleteFile)
if not fScaleFlat:
fScaleFlat = 1.0
xsd.flatScaleFactor = XSDataDouble(fScaleFlat)
return xsd.marshal()
def testGetXSDataRaddoseInput(self):
"""
"""
from XSDataCommon import XSDataLength
from XSDataCommon import XSDataWavelength
from XSDataCommon import XSDataFlux
from XSDataCommon import XSDataSize
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataString
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataTime
from XSDataCommon import XSDataInteger
from XSDataMXv1 import XSDataBeam
from XSDataMXv1 import XSDataStructure
from XSDataMXv1 import XSDataChain
from XSDataMXv1 import XSDataAtom
from XSDataMXv1 import XSDataLigand
from XSDataMXv1 import XSDataCrystal
from XSDataMXv1 import XSDataSpaceGroup
from XSDataMXv1 import XSDataSampleCrystalMM
from XSDataMXv1 import XSDataChemicalCompositionMM
from XSDataMXv1 import XSDataAtomicComposition
from XSDataMXv1 import XSDataSolvent
from XSDataMXv1 import XSDataCell
from EDHandlerXSDataRaddosev10 import EDHandlerXSDataRaddosev10
EDFactoryPluginStatic.loadModule("XSDataRaddosev10")
from XSDataRaddosev10 import XSDataRaddoseInput
xsDataBeam = XSDataBeam()
xsDataBeam.setSize(XSDataSize(x=XSDataLength(0.1),
y=XSDataLength(0.1)))
xsDataBeam.setWavelength(XSDataWavelength(2.41))
xsDataBeam.setFlux(XSDataFlux(1e+12))
xsDataBeam.setExposureTime(XSDataTime(0.037))
xsDataSample = XSDataSampleCrystalMM()
xsDataStructure = XSDataStructure()
xsDataComposition = XSDataChemicalCompositionMM()
xsDataChain = XSDataChain()
xsDataChain.setType(XSDataString("protein"))
xsDataChain.setNumberOfCopies(XSDataDouble(2))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom1 = XSDataAtom()
xsDataAtom1.setSymbol(XSDataString("Se"))
xsDataAtom1.setNumberOf(XSDataDouble(4))
xsDataAtomicComposition.addAtom(xsDataAtom1)
xsDataAtom2 = XSDataAtom()
xsDataAtom2.setSymbol(XSDataString("S"))
xsDataAtom2.setNumberOf(XSDataDouble(5))
xsDataAtomicComposition.addAtom(xsDataAtom2)
xsDataChain.setHeavyAtoms(xsDataAtomicComposition)
xsDataChain.setNumberOfMonomers(XSDataDouble(100))
xsDataStructure.addChain(xsDataChain)
xsDataChain2 = XSDataChain()
xsDataChain2.setType(XSDataString("rna"))
xsDataChain2.setNumberOfCopies(XSDataDouble(1))
xsDataChain2.setNumberOfMonomers(XSDataDouble(60))
xsDataStructure.addChain(xsDataChain2)
xsDataLigand = XSDataLigand()
xsDataLigand.setNumberOfCopies(XSDataDouble(2))
xsDataLigand.setNumberOfLightAtoms(XSDataDouble(42))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom3 = XSDataAtom()
xsDataAtom3.setSymbol(XSDataString("Fe"))
xsDataAtom3.setNumberOf(XSDataDouble(1))
xsDataAtomicComposition.addAtom(xsDataAtom3)
xsDataLigand.setHeavyAtoms(xsDataAtomicComposition)
xsDataStructure.addLigand(xsDataLigand)
xsDataStructure.setNumberOfCopiesInAsymmetricUnit(XSDataDouble(0.25))
xsDataSolvent = XSDataSolvent()
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtomNa = XSDataAtom()
xsDataAtomNa.setSymbol(XSDataString("Na"))
xsDataAtomNa.setConcentration(XSDataDouble(1000))
xsDataAtomicComposition.addAtom(xsDataAtomNa)
xsDataAtomCl = XSDataAtom()
xsDataAtomCl.setSymbol(XSDataString("Cl"))
xsDataAtomCl.setConcentration(XSDataDouble(1000))
xsDataAtomicComposition.addAtom(xsDataAtomCl)
xsDataSolvent.setAtoms(xsDataAtomicComposition)
xsDataComposition.setStructure(xsDataStructure)
xsDataComposition.setSolvent(xsDataSolvent)
xsDataSample.setChemicalComposition(xsDataComposition)
xsDataSample.setSize(
XSDataSize(XSDataLength(0.1), XSDataLength(0.1),
XSDataLength(0.1)))
xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0),
angle_beta=XSDataAngle(90.0),
angle_gamma=XSDataAngle(90.0),
length_a=XSDataLength(78.9),
length_b=XSDataLength(95.162),
length_c=XSDataLength(104.087))
xsDataCrystal = XSDataCrystal()
xsDataSpaceGroup = XSDataSpaceGroup()
xsDataCrystal.setCell(xsDataCell)
xsDataSpaceGroup.setITNumber(XSDataInteger(16))
xsDataCrystal.setSpaceGroup(xsDataSpaceGroup)
xsDataSample.setCrystal(xsDataCrystal)
iNumSymOperators = 4
iNumberOfImages = 2
xsDataRaddosev01Input = EDHandlerXSDataRaddosev10(
).getXSDataRaddoseInput(xsDataBeam, xsDataSample, iNumSymOperators,
iNumberOfImages)
xsDataRaddosev01Input.exportToFile(self.strObtainedInputFile2)
strExpectedInput = EDUtilsTest.readAndParseFile(
self.strReferenceInputFile2)
strObtainedInput = EDUtilsTest.readAndParseFile(
self.strObtainedInputFile2)
xsDataInputExpected = XSDataRaddoseInput.parseString(strExpectedInput)
xsDataInputObtained = XSDataRaddoseInput.parseString(strObtainedInput)
EDAssert.equal(xsDataInputExpected.marshal(),
xsDataInputObtained.marshal())
def testSetDataModelInput(self):
edPluginBest = self.createPlugin()
edConfigurationGood01 = EDConfiguration(
os.path.join(self.strDataPath, "XSConfiguration.xml"))
dictItemGood01 = edConfigurationGood01.get(self.getPluginName())
edPluginBest.setConfig(dictItemGood01, _bLocal=True)
edPluginBest.setScriptExecutable("cat")
edPluginBest.configure()
from XSDataBestv1_3 import XSDataInputBest
xsDataInputBest = XSDataInputBest()
from XSDataCommon import XSDataAbsorbedDoseRate
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataString
from XSDataCommon import XSDataTime
from XSDataCommon import XSDataFile
from XSDataCommon import XSDataAngularSpeed
from XSDataCommon import XSDataString
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataBoolean
xsDataInputBest.setCrystalAbsorbedDoseRate(
XSDataAbsorbedDoseRate(0.22E+06))
xsDataInputBest.setCrystalShape(XSDataDouble(1))
xsDataInputBest.setCrystalSusceptibility(XSDataDouble(1.5))
xsDataInputBest.setDetectorType(XSDataString("q210-2x"))
xsDataInputBest.setBeamExposureTime(XSDataTime(1))
xsDataInputBest.setBeamMaxExposureTime(XSDataTime(10000))
xsDataInputBest.setBeamMinExposureTime(XSDataTime(0.1))
xsDataInputBest.setGoniostatMinRotationWidth(XSDataAngle(0.1))
xsDataInputBest.setGoniostatMaxRotationSpeed(XSDataAngularSpeed(10))
xsDataInputBest.setAimedResolution(XSDataDouble(2))
xsDataInputBest.setAimedRedundancy(XSDataDouble(6.5))
xsDataInputBest.setAimedCompleteness(XSDataDouble(0.9))
xsDataInputBest.setAimedIOverSigma(XSDataDouble(3))
xsDataInputBest.setComplexity(XSDataString("min"))
xsDataInputBest.setAnomalousData(XSDataBoolean(False))
fileDirectory = edPluginBest.getWorkingDirectory()
bestFileContentDat = EDUtilsFile.readFile(
os.path.join(self.strDataPath, "bestfile.dat"))
xsDataInputBest.setBestFileContentDat(XSDataString(bestFileContentDat))
bestFileContentPar = EDUtilsFile.readFile(
os.path.join(self.strDataPath, "bestfile.par"))
xsDataInputBest.setBestFileContentPar(XSDataString(bestFileContentPar))
bestFileContentHKL = EDUtilsFile.readFile(
os.path.join(self.strDataPath, "bestfile1.hkl"))
xsDataInputBest.addBestFileContentHKL(XSDataString(bestFileContentHKL))
xsDataInputBest.exportToFile(self.obtainedInputFile)
pyStrExpectedInput = self.readAndParseFile(self.referenceInputFile)
pyStrObtainedInput = self.readAndParseFile(self.obtainedInputFile)
xsDataInputExpected = XSDataInputBest.parseString(pyStrExpectedInput)
xsDataInputObtained = XSDataInputBest.parseString(pyStrObtainedInput)
EDAssert.equal(xsDataInputExpected.marshal(),
xsDataInputObtained.marshal())
EDUtilsFile.deleteFile(self.obtainedInputFile)
self.cleanUp(edPluginBest)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_2.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
shutil.copyfile(self.lstInput[0].path.value,
self.dataInput.mergedCurve.path.value)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], self.lstInput[1]])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.lstSummary.append("WARNING: " + self.strRadiationDamage)
self.lstSummary.append("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
#outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
self.dictAve[self.tKey] = self.dataInput.mergedCurve
if len(self.tKey) == 3 and (self.tKey[0], self.tKey[1],
0.0) in self.dictAve:
#try to subtract buffer automatically
bufferKey = (self.tKey[0], self.tKey[1], 0.0)
bufferPath = self.dictAve[bufferKey]
if self.strSubFile is None:
self.strSubFile = self.strMergedFile.replace(
"_ave.dat", "_sub.dat")
if self.strMergedFile == self.strSubFile:
self.strSubFile = None
return
xsdSubFile = XSDataFile(XSDataString(self.strSubFile))
xsdSub = XSDataInputDatop(
inputCurve=[self.dataInput.mergedCurve, bufferPath],
outputCurve=xsdSubFile,
operation=XSDataString("SUB"))
self.lstSub.append("Merged data from : %s" %
self.dataInput.mergedCurve.path.value)
self.lstSub.append("Subtracted buffer: %s" %
bufferPath.path.value)
self.__edPluginExecDataop = self.loadPlugin(
self.__strControlledPluginDatop)
self.__edPluginExecDataop.setDataInput(xsdSub)
self.__edPluginExecDataop.connectSUCCESS(
self.doSuccessExecDatop)
self.__edPluginExecDataop.connectFAILURE(
self.doFailureExecDatop)
self.__edPluginExecDataop.executeSynchronous()
if self.isFailure():
return
elif self.strSubFile is not None:
self.__edPluginExecAutoRg = self.loadPlugin(
self.__strControlledPluginAutoRG)
xsd = XSDataInputAutoRg(inputCurve=[xsdSubFile])
self.__edPluginExecAutoRg.setDataInput(xsd)
self.__edPluginExecAutoRg.connectSUCCESS(
self.doSuccessExecAutoRg)
self.__edPluginExecAutoRg.connectFAILURE(
self.doFailureExecAutoRg)
self.__edPluginExecAutoRg.executeSynchronous()
self.rewriteHeader(self.strSubFile, self.strSubFile)
else:
self.__edPluginExecAutoRg = self.loadPlugin(
self.__strControlledPluginAutoRG)
xsd = XSDataInputAutoRg(
inputCurve=[self.dataInput.mergedCurve])
self.__edPluginExecAutoRg.setDataInput(xsd)
self.__edPluginExecAutoRg.connectSUCCESS(
self.doSuccessExecAutoRg)
self.__edPluginExecAutoRg.connectFAILURE(
self.doFailureExecAutoRg)
self.__edPluginExecAutoRg.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_0.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
# if self.isFailure():
# return
if len(self.lstInput) == 1:
shutil.copyfile(self.lstInput[0].path.value,
self.dataInput.mergedCurve.path.value)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for oneFile in self.lstInput[1:]:
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity
is not None) and (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstSummary.append("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
def setPluginInput(self, _edPlugin):
xsDataDiffractionPlan = XSDataDiffractionPlan()
if (not self.__fMaxExposureTimePerDataCollection is None):
xsDataDiffractionPlan.setMaxExposureTimePerDataCollection(
XSDataTime(self.__fMaxExposureTimePerDataCollection))
if (not self.__strForcedSpaceGroup is None):
xsDataDiffractionPlan.setForcedSpaceGroup(
XSDataString(self.__strForcedSpaceGroup))
if (not self.__bAnomalousData is None):
xsDataDiffractionPlan.setAnomalousData(
XSDataBoolean(self.__bAnomalousData))
if (not self.__strStrategyOption is None):
xsDataDiffractionPlan.setStrategyOption(
XSDataString(self.__strStrategyOption))
if (not self.__fDoseLimit is None):
xsDataDiffractionPlan.setDoseLimit(XSDataDouble(self.__fDoseLimit))
if (not self.__fAimedCompleteness is None):
xsDataDiffractionPlan.setAimedCompleteness(
XSDataDouble(self.__fAimedCompleteness))
if (not self.__fAimedMultiplicity is None):
xsDataDiffractionPlan.setAimedMultiplicity(
XSDataDouble(self.__fAimedMultiplicity))
if (not self.__fAimedResolution is None):
xsDataDiffractionPlan.setAimedResolution(
XSDataDouble(self.__fAimedResolution))
if (not self.__strComplexity is None):
xsDataDiffractionPlan.setComplexity(
XSDataString(self.__strComplexity))
_edPlugin.setDataInput(xsDataDiffractionPlan, "diffractionPlan")
if (not self.__listImagePaths is None):
for strImagePath in self.__listImagePaths:
_edPlugin.setDataInput(XSDataString(strImagePath),
"imagePaths")
if (not self.__xsDataInputCharacterisation is None):
_edPlugin.setDataInput(self.__xsDataInputCharacterisation,
"inputCharacterisation")
if (not self.__fFlux is None):
_edPlugin.setDataInput(XSDataFloat(self.__fFlux), "flux")
if (not self.__fMinExposureTimePerImage is None):
_edPlugin.setDataInput(
XSDataFloat(self.__fMinExposureTimePerImage),
"minExposureTimePerImage")
if (not self.__fBeamSize is None):
_edPlugin.setDataInput(XSDataFloat(self.__fBeamSize), "beamSize")
if (not self.__bTemplateMode is None):
_edPlugin.setDataInput(XSDataBoolean(self.__bTemplateMode),
"templateMode")
if (not self.__strGeneratedTemplateFile is None):
_edPlugin.setDataInput(
XSDataString(self.__strGeneratedTemplateFile),
"generatedTemplateFile")
if (not self.__strResultsFilePath is None):
_edPlugin.setDataInput(XSDataString(self.__strResultsFilePath),
"resultsFilePath")
if (not self.__fBeamPosX is None):
_edPlugin.setDataInput(XSDataFloat(self.__fBeamPosX), "beamPosX")
if (not self.__fBeamPosY is None):
_edPlugin.setDataInput(XSDataFloat(self.__fBeamPosY), "beamPosY")
if (not self.__iDataCollectionId is None):
_edPlugin.setDataInput(XSDataInteger(self.__iDataCollectionId),
"dataCollectionId")
if (not self.__strShortComments is None):
_edPlugin.setDataInput(XSDataString(self.__strShortComments),
"shortComments")
if (not self.__strComments is None):
_edPlugin.setDataInput(XSDataString(self.__strComments),
"comments")
if (not self.__fTransmission is None):
_edPlugin.setDataInput(XSDataDouble(self.__fTransmission),
"transmission")
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("EDPluginExecReadImageHeaderPilatus6Mv10.process")
xsDataInputReadImageHeader = self.getDataInput()
xsDataFile = xsDataInputReadImageHeader.getImage()
strPath = xsDataFile.getPath().getValue()
dictPilatus6MHeader = self.readHeaderPilatus6M(strPath)
if (dictPilatus6MHeader is None):
strErrorMessage = "EDPluginExecReadImageHeaderPilatus6Mv10.process : Cannot read header : %s" % strPath
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
else:
xsDataExperimentalCondition = XSDataExperimentalCondition()
xsDataDetector = XSDataDetector()
iNoPixelsX = 2463
iNoPixelsY = 2527
xsDataDetector.setNumberPixelX(XSDataInteger(iNoPixelsX))
xsDataDetector.setNumberPixelY(XSDataInteger(iNoPixelsY))
# Pixel size
listPixelSizeXY = dictPilatus6MHeader["Pixel_size"].split(" ")
fPixelSizeX = float(listPixelSizeXY[0]) * 1000
xsDataDetector.setPixelSizeX(XSDataLength(fPixelSizeX))
fPixelSizeY = float(listPixelSizeXY[3]) * 1000
xsDataDetector.setPixelSizeY(XSDataLength(fPixelSizeY))
# Beam position
listBeamPosition = dictPilatus6MHeader["Beam_xy"].replace(
"(", " ").replace(")", " ").replace(",", " ").split()
fBeamPositionX = float(listBeamPosition[1]) * fPixelSizeX
fBeamPositionY = float(listBeamPosition[0]) * fPixelSizeY
xsDataDetector.setBeamPositionX(XSDataLength(fBeamPositionX))
xsDataDetector.setBeamPositionY(XSDataLength(fBeamPositionY))
fDistance = float(
dictPilatus6MHeader["Detector_distance"].split(" ")[0]) * 1000
xsDataDetector.setDistance(XSDataLength(fDistance))
# xsDataDetector.setNumberBytesInHeader(XSDataInteger(float(dictPilatus6MHeader[ "header_size" ])))
xsDataDetector.setSerialNumber(
XSDataString(dictPilatus6MHeader["Detector:"]))
# #xsDataDetector.setBin( XSDataString( dictPilatus6MHeader[ "BIN" ] ) ) )
# #xsDataDetector.setDataType( XSDataString( dictPilatus6MHeader[ "TYPE" ] ) ) )
# #xsDataDetector.setByteOrder( XSDataString( dictPilatus6MHeader[ "BYTE_ORDER" ] ) ) )
# xsDataDetector.setImageSaturation(XSDataInteger(int(dictPilatus6MHeader[ "saturation_level" ])))
xsDataDetector.setName(XSDataString("PILATUS 6M"))
xsDataDetector.setType(XSDataString("pilatus6m"))
xsDataExperimentalCondition.setDetector(xsDataDetector)
# Beam object
xsDataBeam = XSDataBeam()
xsDataBeam.setWavelength(
XSDataWavelength(
float(dictPilatus6MHeader["Wavelength"].split(" ")[0])))
xsDataBeam.setExposureTime(
XSDataTime(
float(dictPilatus6MHeader["Exposure_time"].split(" ")[0])))
xsDataExperimentalCondition.setBeam(xsDataBeam)
# Goniostat object
xsDataGoniostat = XSDataGoniostat()
fRotationAxisStart = float(
dictPilatus6MHeader["Start_angle"].split(" ")[0])
fOscillationWidth = float(
dictPilatus6MHeader["Angle_increment"].split(" ")[0])
if "Kappa" in dictPilatus6MHeader.keys():
fKappa = float(dictPilatus6MHeader["Kappa"].split(" ")[0])
xsDataGoniostat.setKappa(XSDataAngle(fKappa))
if "Phi" in dictPilatus6MHeader.keys():
fPhi = float(dictPilatus6MHeader["Phi"].split(" ")[0])
xsDataGoniostat.setPhi(XSDataAngle(fPhi))
xsDataGoniostat.setRotationAxisStart(
XSDataAngle(fRotationAxisStart))
xsDataGoniostat.setRotationAxisEnd(
XSDataAngle(fRotationAxisStart + fOscillationWidth))
xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth))
xsDataExperimentalCondition.setGoniostat(xsDataGoniostat)
#
# Create the image object
xsDataImage = XSDataImage()
xsDataImage.setPath(XSDataString(strPath))
if "DateTime" in dictPilatus6MHeader:
strTimeStamp = dictPilatus6MHeader["DateTime"]
xsDataImage.setDate(XSDataString(strTimeStamp))
iImageNumber = EDUtilsImage.getImageNumber(strPath)
xsDataImage.setNumber(XSDataInteger(iImageNumber))
xsDataSubWedge = XSDataSubWedge()
xsDataSubWedge.setExperimentalCondition(
xsDataExperimentalCondition)
xsDataSubWedge.addImage(xsDataImage)
self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader()
self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
def testSetDataModelInput(self):
"""
"""
edPluginBest = self.createPlugin()
xsPluginItemGood01 = self.getPluginConfiguration(
os.path.join(self.strDataPath, "XSConfiguration.xml"))
edPluginBest.setConfiguration(xsPluginItemGood01)
edPluginBest.setScriptExecutable("cat")
edPluginBest.configure()
from XSDataBestv1_1 import XSDataInputBest
xsDataInputBest = XSDataInputBest()
from XSDataCommon import XSDataAbsorbedDoseRate
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataTime
from XSDataCommon import XSDataSpeed
from XSDataCommon import XSDataString
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataBoolean
xsDataInputBest.setCrystalAbsorbedDoseRate(
XSDataAbsorbedDoseRate(0.22E+06))
xsDataInputBest.setCrystalShape(XSDataDouble(1))
xsDataInputBest.setCrystalSusceptibility(XSDataDouble(1.5))
xsDataInputBest.setDetectorType(XSDataString("q210-2x"))
xsDataInputBest.setBeamExposureTime(XSDataTime(1))
xsDataInputBest.setBeamMaxExposureTime(XSDataTime(10000))
xsDataInputBest.setBeamMinExposureTime(XSDataTime(0.1))
xsDataInputBest.setGoniostatMinRotationWidth(XSDataAngle(0.1))
xsDataInputBest.setGoniostatMaxRotationSpeed(XSDataSpeed(10))
xsDataInputBest.setAimedResolution(XSDataDouble(2))
xsDataInputBest.setAimedRedundancy(XSDataDouble(6.5))
xsDataInputBest.setAimedCompleteness(XSDataDouble(0.9))
xsDataInputBest.setAimedIOverSigma(XSDataDouble(3))
xsDataInputBest.setComplexity(XSDataString("min"))
xsDataInputBest.setAnomalousData(XSDataBoolean(True))
bestFileContentDat = EDUtilsFile.readFile(
os.path.join(self.strDataPath, "bestfile.dat"))
xsDataInputBest.setBestFileContentDat(XSDataString(bestFileContentDat))
bestFileContentPar = EDUtilsFile.readFile(
os.path.join(self.strDataPath, "bestfile.par"))
xsDataInputBest.setBestFileContentPar(XSDataString(bestFileContentPar))
bestFileContentHKL = EDUtilsFile.readFile(
os.path.join(self.strDataPath, "bestfile1.hkl"))
listBestFileContentHKL = []
listBestFileContentHKL.append(XSDataString(bestFileContentHKL))
xsDataInputBest.setBestFileContentHKL(listBestFileContentHKL)
xsDataInputBest.outputFile(self.strObtainedInputFile)
strExpectedInput = self.readAndParseFile(self.strReferenceInputFile)
strObtainedInput = self.readAndParseFile(self.strObtainedInputFile)
xsDataInputExpected = XSDataInputBest.parseString(strExpectedInput)
xsDataInputObtained = XSDataInputBest.parseString(strObtainedInput)
EDAssert.equal(xsDataInputExpected.marshal(),
xsDataInputObtained.marshal())
os.remove(self.strObtainedInputFile)
self.cleanUp(edPluginBest)
def testSetDataModelInput(self):
"""
"""
edPluginStrategy = self.createPlugin()
xsPluginItemGood01 = self.getPluginConfiguration(os.path.join(self.strDataPath, "XSConfiguration_ESRF.xml"))
edPluginStrategy.setConfiguration(xsPluginItemGood01)
edPluginStrategy.configure()
from XSDataMXv1 import XSDataStrategyInput
xSDataStrategy = XSDataStrategyInput()
# Beam
from XSDataCommon import XSDataFlux
from XSDataCommon import XSDataWavelength
from XSDataCommon import XSDataSize
from XSDataCommon import XSDataLength
from XSDataCommon import XSDataTime
from XSDataMXv1 import XSDataBeam
from XSDataMXv1 import XSDataExperimentalCondition
xsExperimentalCondition = XSDataExperimentalCondition()
xsBeam = XSDataBeam()
xsBeam.setFlux(XSDataFlux(1e+12))
xsBeam.setWavelength(XSDataWavelength(2.41))
xsBeam.setSize(XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1)))
xsBeam.setExposureTime(XSDataTime(1))
xsExperimentalCondition.setBeam(xsBeam)
# Detector and Exposure Time
from XSDataMXv1 import XSDataDetector
from XSDataCommon import XSDataString
from XSDataMXv1 import XSDataGoniostat
xsDataDetector = XSDataDetector()
xsDataDetector.setType(XSDataString("q210-2x"))
xsExperimentalCondition.setDetector(xsDataDetector)
xsDataGoniostat = XSDataGoniostat()
xsDataGoniostat.setRotationAxis(XSDataString("phi"))
xsExperimentalCondition.setGoniostat(xsDataGoniostat)
xSDataStrategy.setExperimentalCondition(xsExperimentalCondition)
# Best Files
bestFileContentDat = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile.dat"))
xSDataStrategy.setBestFileContentDat(XSDataString(bestFileContentDat))
bestFileContentPar = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile.par"))
xSDataStrategy.setBestFileContentPar(XSDataString(bestFileContentPar))
bestFileContentHKL = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile1.hkl"))
listBestFileContentHKL = []
listBestFileContentHKL.append(XSDataString(bestFileContentHKL))
xSDataStrategy.setBestFileContentHKL(listBestFileContentHKL)
# Crystal
from XSDataCommon import XSDataFloat
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataInteger
from XSDataMXv1 import XSDataCrystal
from XSDataMXv1 import XSDataStructure
from XSDataMXv1 import XSDataChain
from XSDataMXv1 import XSDataAtom
from XSDataMXv1 import XSDataLigand
from XSDataMXv1 import XSDataSampleCrystalMM
from XSDataMXv1 import XSDataChemicalCompositionMM
from XSDataMXv1 import XSDataAtomicComposition
from XSDataMXv1 import XSDataSolvent
from XSDataMXv1 import XSDataCell
from XSDataMXv1 import XSDataSpaceGroup
xsDataSampleCrystalMM = XSDataSampleCrystalMM()
xsDataStructure = XSDataStructure()
xsDataComposition = XSDataChemicalCompositionMM()
xsDataChain = XSDataChain()
xsDataChain.setType(XSDataString("protein"))
xsDataChain.setNumberOfCopies(XSDataFloat(2))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom1 = XSDataAtom()
xsDataAtom1.setSymbol(XSDataString("Se"))
xsDataAtom1.setNumberOf(XSDataFloat(4))
xsDataAtomicComposition.addAtom(xsDataAtom1)
xsDataChain.setHeavyAtoms(xsDataAtomicComposition)
xsDataChain.setNumberOfMonomers(XSDataFloat(100))
xsDataStructure.addChain(xsDataChain)
xsDataChain2 = XSDataChain()
xsDataChain2.setType(XSDataString("rna"))
xsDataChain2.setNumberOfCopies(XSDataFloat(1))
xsDataChain2.setNumberOfMonomers(XSDataFloat(60))
xsDataStructure.addChain(xsDataChain2)
xsDataLigand = XSDataLigand()
xsDataLigand.setNumberOfCopies(XSDataFloat(2))
xsDataLigand.setNumberOfLightAtoms(XSDataFloat(42))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom2 = XSDataAtom()
xsDataAtom2.setSymbol(XSDataString("Fe"))
xsDataAtom2.setNumberOf(XSDataFloat(1))
xsDataAtomicComposition.addAtom(xsDataAtom2)
xsDataLigand.setHeavyAtoms(xsDataAtomicComposition)
xsDataStructure.addLigand(xsDataLigand)
xsDataStructure.setNumberOfCopiesInAsymmetricUnit(XSDataFloat(0.25))
xsDataSolvent = XSDataSolvent()
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom3 = XSDataAtom()
xsDataAtom3.setSymbol(XSDataString("Na"))
xsDataAtom3.setConcentration(XSDataFloat(1000))
xsDataAtom4 = XSDataAtom()
xsDataAtom4.setSymbol(XSDataString("Cl"))
xsDataAtom4.setConcentration(XSDataFloat(1000))
xsDataAtomicComposition.addAtom(xsDataAtom3)
xsDataAtomicComposition.addAtom(xsDataAtom4)
xsDataSolvent.setAtoms(xsDataAtomicComposition)
xsDataComposition.setStructure(xsDataStructure)
xsDataComposition.setSolvent(xsDataSolvent)
xsDataSampleCrystalMM.setChemicalComposition(xsDataComposition)
xsDataSampleCrystalMM.setSize(XSDataSize(XSDataLength(0.1), XSDataLength(0.1), XSDataLength(0.1)))
xsDataCrystal = XSDataCrystal()
xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0),
angle_beta=XSDataAngle(90.0),
angle_gamma=XSDataAngle(90.0),
length_a=XSDataLength(78.9),
length_b=XSDataLength(95.162),
length_c=XSDataLength(104.087))
xsDataCrystal.setCell(xsDataCell)
xsDataSpaceGroup = XSDataSpaceGroup()
xsDataSpaceGroup.setITNumber(XSDataInteger(16))
xsDataCrystal.setSpaceGroup(xsDataSpaceGroup)
xsDataSampleCrystalMM.setSusceptibility(XSDataFloat(1.5))
xSDataStrategy.setCrystalRefined(xsDataCrystal)
xSDataStrategy.setSample(xsDataSampleCrystalMM)
xSDataStrategy.outputFile(self.strObtainedInputFile)
strExpectedInput = self.readAndParseFile (self.strReferenceInputFile)
strObtainedInput = self.readAndParseFile (self.strObtainedInputFile)
xsDataInputExpected = XSDataStrategyInput.parseString(strExpectedInput)
xsDataInputObtained = XSDataStrategyInput.parseString(strObtainedInput)
EDAssert.equal(xsDataInputExpected.marshal(), xsDataInputObtained.marshal())
def getDataCollectionOutputDataFromLog(self, _strBestLog):
xsDataResultBest = XSDataResultBest()
# loop through the lines of the log file
isScanningWedge = False
indexLine = 0
listLog = _strBestLog.split("\n")
iCollectionPlanNumber = 1
while indexLine < len(listLog):
if "Plan of data collection for radiation damage characterisation" in listLog[
indexLine]:
xsDataBestCollectionPlan = XSDataBestCollectionPlan()
xsDataBestStrategySummary = XSDataBestStrategySummary()
indexLine += 2
regEx = re.compile(
""" Resolution limit =\s*(\d+\.\d+) Angstrom\s*Distance =\s*(\d+\.\d+)mm"""
)
matchObj = regEx.search(listLog[indexLine])
if matchObj == None:
raise BaseException("Cannot parse best log file!")
resultMatch = matchObj.groups()
resultion = float(resultMatch[0])
distance = float(resultMatch[1])
xsDataBestStrategySummary.resolution = XSDataDouble(resultion)
xsDataBestStrategySummary.distance = XSDataLength(distance)
xsDataBestCollectionPlan.strategySummary = xsDataBestStrategySummary
# Burn strategy
indexLine += 7
collectionRunNumber = 1
while not listLog[indexLine].startswith(
"-------------------------------"):
listLine = listLog[indexLine].split()
if listLine[0].startswith(
"exposure") or listLine[0].startswith("burn"):
xsDataBestCollectionRun = XSDataBestCollectionRun()
xsDataBestCollectionRun.collectionRunNumber = XSDataInteger(
collectionRunNumber)
xsDataBestCollectionRun.action = XSDataString(
listLine[0])
xsDataBestCollectionRun.phiStart = XSDataAngle(
listLine[1])
xsDataBestCollectionRun.phiWidth = XSDataAngle(
listLine[2])
xsDataBestCollectionRun.exposureTime = XSDataTime(
listLine[3])
xsDataBestCollectionRun.numberOfImages = XSDataInteger(
listLine[4].split("|")[0])
xsDataBestCollectionRun.transmission = XSDataDouble(
listLine[5])
xsDataBestCollectionPlan.addCollectionRun(
xsDataBestCollectionRun)
collectionRunNumber += 1
indexLine += 1
xsDataBestCollectionPlan.collectionPlanNumber = XSDataInteger(
iCollectionPlanNumber)
xsDataResultBest.addCollectionPlan(xsDataBestCollectionPlan)
if "Main Wedge" in listLog[indexLine] or "Low resolution Wedge" in listLog[indexLine] \
or "Strategy for SAD data collection" in listLog[indexLine]:
isScanningWedge = True
xsDataBestCollectionPlan = XSDataBestCollectionPlan()
xsDataBestStrategySummary = XSDataBestStrategySummary()
if "Main Wedge" in listLog[indexLine]:
indexLine += 2
# ResolutionReasoning
xsDataBestStrategySummary.resolutionReasoning = XSDataString(
listLog[indexLine].strip())
elif "Strategy for SAD data collection" in listLog[indexLine]:
indexLine += 2
xsDataBestStrategySummary.resolutionReasoning = XSDataString(
listLog[indexLine].strip())
else:
xsDataBestStrategySummary.resolutionReasoning = XSDataString(
"Low-resolution pass, no overloads and full completeness"
)
# Resolution, transmission, distance
indexLine += 1
while not "Resolution limit" in listLog[indexLine]:
indexLine += 1
regEx = re.compile(
""" Resolution limit =\s*(\d+\.\d+) Angstrom Transmission =\s*(\d+\.\d+)% Distance =\s*(\d+\.\d+)mm"""
)
matchObj = regEx.search(listLog[indexLine])
if matchObj == None:
raise BaseException("Cannot parse best log file!")
resultMatch = matchObj.groups()
resultion = float(resultMatch[0])
transmission = float(resultMatch[1])
distance = float(resultMatch[2])
xsDataBestStrategySummary.resolution = XSDataDouble(resultion)
xsDataBestStrategySummary.transmission = XSDataDouble(
transmission)
xsDataBestStrategySummary.distance = XSDataLength(distance)
# Resolution, transmission, distance
indexLine += 8
while not listLog[indexLine].startswith(
"-------------------------------"):
listLine = listLog[indexLine].split()
xsDataBestCollectionRun = XSDataBestCollectionRun()
xsDataBestCollectionRun.collectionRunNumber = XSDataInteger(
listLine[0])
xsDataBestCollectionRun.phiStart = XSDataAngle(listLine[1])
xsDataBestCollectionRun.phiWidth = XSDataAngle(listLine[2])
xsDataBestCollectionRun.exposureTime = XSDataTime(
listLine[3])
xsDataBestCollectionRun.numberOfImages = XSDataInteger(
listLine[4].split("|")[0])
xsDataBestCollectionRun.overlaps = XSDataString(
listLine[5])
xsDataBestCollectionPlan.addCollectionRun(
xsDataBestCollectionRun)
indexLine += 1
#
xsDataBestStrategySummary.completeness = XSDataDouble(
listLine[11])
#
while not "Redundancy" in listLog[indexLine]:
indexLine += 1
xsDataBestStrategySummary.redundancy = XSDataDouble(
listLog[indexLine].split()[-1])
#
while not "I/Sigma (outer shell)" in listLog[indexLine]:
indexLine += 1
xsDataBestStrategySummary.iSigma = XSDataDouble(
listLog[indexLine].split()[6].replace(")", ""))
#
while not "Total Exposure time" in listLog[indexLine]:
indexLine += 1
xsDataBestStrategySummary.totalExposureTime = XSDataTime(
listLog[indexLine].split()[4])
#
while not "Total Data Collection time" in listLog[indexLine]:
indexLine += 1
xsDataBestStrategySummary.totalDataCollectionTime = XSDataTime(
listLog[indexLine].split()[5])
#
while not "Wedge Data Collection Statistics according to the Strategy" in listLog[
indexLine]:
indexLine += 1
(xsDataBestStatisticalPrediction,
indexLine) = self.getXSDataBestStatisticalPrediction(
listLog, indexLine)
xsDataBestCollectionPlan.strategySummary = xsDataBestStrategySummary
xsDataBestCollectionPlan.statisticalPrediction = xsDataBestStatisticalPrediction
xsDataBestCollectionPlan.collectionPlanNumber = XSDataInteger(
iCollectionPlanNumber)
iCollectionPlanNumber += 1
xsDataResultBest.addCollectionPlan(xsDataBestCollectionPlan)
if "Additional information" in listLog[indexLine]:
#
while not "Relative scale" in listLog[indexLine]:
indexLine += 1
scale = float(listLog[indexLine].split()[-1])
#
while not "Overall B-factor" in listLog[indexLine]:
indexLine += 1
#
bFactor = float(listLog[indexLine].split()[-2])
while not "Estimated limit of resolution" in listLog[indexLine]:
indexLine += 1
rankingResolution = float(listLog[indexLine].split()[5])
for collectionPlan in xsDataResultBest.collectionPlan:
xsDataCrystalScale = XSDataCrystalScale()
xsDataCrystalScale.scale = XSDataDouble(scale)
xsDataCrystalScale.bFactor = XSDataDouble(bFactor)
collectionPlan.crystalScale = xsDataCrystalScale
collectionPlan.strategySummary.rankingResolution = XSDataDouble(
rankingResolution)
indexLine += 1
return xsDataResultBest
