def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("*** EDPluginExecReadImageHeaderADSCv10.process")
xsDataInputReadImageHeader = self.getDataInput()
xsDataFile = xsDataInputReadImageHeader.getImage()
strPath = xsDataFile.getPath().getValue()
strAbsolutePath = os.path.abspath(strPath)
dictHeader = self.readHeaderADSC(strPath)
if (dictHeader is None):
strErrorMessage = "EDPluginExecReadImageHeaderADSCv10.process : error when reading header from %s" % strAbsolutePath
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
else:
xsDataExperimentalCondition = XSDataExperimentalCondition()
xsDataDetector = XSDataDetector()
xsDataDetector.setBeamPositionX(
XSDataLength(float(dictHeader["BEAM_CENTER_X"])))
xsDataDetector.setBeamPositionY(
XSDataLength(float(dictHeader["BEAM_CENTER_Y"])))
xsDataDetector.setDistance(
XSDataLength(float(dictHeader["DISTANCE"])))
fPixelSize = float(dictHeader["PIXEL_SIZE"])
xsDataDetector.setPixelSizeX(XSDataLength(fPixelSize))
xsDataDetector.setPixelSizeY(XSDataLength(fPixelSize))
if "TWOTHETA" in dictHeader.keys():
xsDataDetector.setTwoTheta(
XSDataAngle(float(dictHeader["TWOTHETA"])))
xsDataDetector.setNumberBytesInHeader(
XSDataInteger(float(dictHeader["HEADER_BYTES"])))
xsDataDetector.setSerialNumber(
XSDataString(dictHeader["DETECTOR_SN"]))
xsDataDetector.setNumberPixelX(
XSDataInteger(int(dictHeader["SIZE1"])))
xsDataDetector.setNumberPixelY(
XSDataInteger(int(dictHeader["SIZE2"])))
xsDataDetector.setBin(XSDataString(dictHeader["BIN"]))
xsDataDetector.setDataType(XSDataString(dictHeader["TYPE"]))
xsDataDetector.setByteOrder(XSDataString(dictHeader["BYTE_ORDER"]))
if "CCD_IMAGE_SATURATION" in dictHeader.keys():
xsDataDetector.setImageSaturation(
XSDataInteger(int(dictHeader["CCD_IMAGE_SATURATION"])))
# Determine type of detector...
iNoPixelsX = xsDataDetector.getNumberPixelX().getValue()
iNoPixelsY = xsDataDetector.getNumberPixelY().getValue()
if (iNoPixelsX == 2304 and iNoPixelsY == 2304):
xsDataDetector.setName(XSDataString("ADSC Q4"))
xsDataDetector.setType(XSDataString("q4"))
elif (iNoPixelsX == 1152 and iNoPixelsY == 1152):
xsDataDetector.setName(XSDataString("ADSC Q4 bin 2x2"))
xsDataDetector.setType(XSDataString("q4-2x"))
elif (iNoPixelsX == 4096 and iNoPixelsY == 4096):
xsDataDetector.setName(XSDataString("ADSC Q210"))
xsDataDetector.setType(XSDataString("q210"))
elif (iNoPixelsX == 2048 and iNoPixelsY == 2048):
xsDataDetector.setName(XSDataString("ADSC Q210 bin 2x2"))
xsDataDetector.setType(XSDataString("q210-2x"))
elif (iNoPixelsX == 6144 and iNoPixelsY == 6144):
xsDataDetector.setName(XSDataString("ADSC Q315"))
xsDataDetector.setType(XSDataString("q315"))
elif (iNoPixelsX == 3072 and iNoPixelsY == 3072):
xsDataDetector.setName(XSDataString("ADSC Q315 bin 2x2"))
xsDataDetector.setType(XSDataString("q315-2x"))
else:
strErrorMessage = EDMessage.ERROR_DATA_HANDLER_02 % (
"EDPluginExecReadImageHeaderADSCv10.process",
"Unknown detector type")
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
raise RuntimeError(strErrorMessage)
xsDataExperimentalCondition.setDetector(xsDataDetector)
# Beam object
xsDataBeam = XSDataBeam()
xsDataBeam.setWavelength(
XSDataWavelength(float(dictHeader["WAVELENGTH"])))
xsDataBeam.setExposureTime(XSDataTime(float(dictHeader["TIME"])))
xsDataExperimentalCondition.setBeam(xsDataBeam)
# Goniostat object
xsDataGoniostat = XSDataGoniostat()
fRotationAxisStart = float(dictHeader["OSC_START"])
fOscillationWidth = float(dictHeader["OSC_RANGE"])
xsDataGoniostat.setRotationAxisStart(
XSDataAngle(fRotationAxisStart))
xsDataGoniostat.setRotationAxisEnd(
XSDataAngle(fRotationAxisStart + fOscillationWidth))
xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth))
strRotationAxis = None
if ("AXIS" in dictHeader.keys()):
strRotationAxis = dictHeader["AXIS"]
elif ("OSC_AXIS" in dictHeader.keys()):
strRotationAxis = dictHeader["OSC_AXIS"]
else:
strErrorMessage = "EDPluginExecReadImageHeaderADSCv10.process : Neither AXIS nor OSC_AXIS header item found."
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
xsDataGoniostat.setRotationAxis(XSDataString(strRotationAxis))
xsDataExperimentalCondition.setGoniostat(xsDataGoniostat)
# Create the image object
xsDataImage = XSDataImage()
xsDataImage.setPath(XSDataString(strAbsolutePath))
xsDataImage.setDate(XSDataString(dictHeader["DATE"]))
strFileName = os.path.basename(strPath)
iImageNumber = EDUtilsImage.getImageNumber(strFileName)
xsDataImage.setNumber(XSDataInteger(iImageNumber))
xsDataSubWedge = XSDataSubWedge()
xsDataSubWedge.setExperimentalCondition(
xsDataExperimentalCondition)
xsDataSubWedge.addImage(xsDataImage)
self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader()
self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
def input_from_params(self, data_collection, char_params):
edna_input = XSDataInputMXCuBE.parseString(self.edna_default_input)
if data_collection.id:
edna_input.setDataCollectionId(XSDataInteger(data_collection.id))
# Beam object
beam = edna_input.getExperimentalCondition().getBeam()
try:
transmission = HWR.beamline.transmission.get_value()
beam.setTransmission(XSDataDouble(transmission))
except AttributeError:
import traceback
logging.getLogger("HWR").debug(
"EDNACharacterisation. transmission not saved ")
logging.getLogger("HWR").debug(traceback.format_exc())
try:
wavelength = HWR.beamline.energy.get_wavelength()
beam.setWavelength(XSDataWavelength(wavelength))
except AttributeError:
pass
try:
beam.setFlux(XSDataFlux(HWR.beamline.flux.get_value()))
except AttributeError:
pass
try:
min_exp_time = self.collect_obj.detector_hwobj.get_exposure_time_limits(
)[0]
beam.setMinExposureTimePerImage(XSDataTime(min_exp_time))
except AttributeError:
pass
try:
beamsize = self.collect_obj.beam_info_hwobj.get_beam_size()
if None not in beamsize:
beam.setSize(
XSDataSize(
x=XSDataLength(float(beamsize[0])),
y=XSDataLength(float(beamsize[1])),
))
except AttributeError:
pass
# Optimization parameters
diff_plan = edna_input.getDiffractionPlan()
aimed_i_sigma = XSDataDouble(char_params.aimed_i_sigma)
aimed_completness = XSDataDouble(char_params.aimed_completness)
aimed_multiplicity = XSDataDouble(char_params.aimed_multiplicity)
aimed_resolution = XSDataDouble(char_params.aimed_resolution)
complexity = char_params.strategy_complexity
complexity = XSDataString(qme.STRATEGY_COMPLEXITY[complexity])
permitted_phi_start = XSDataAngle(char_params.permitted_phi_start)
_range = char_params.permitted_phi_end - char_params.permitted_phi_start
rotation_range = XSDataAngle(_range)
if char_params.aimed_i_sigma:
diff_plan.setAimedIOverSigmaAtHighestResolution(aimed_i_sigma)
if char_params.aimed_completness:
diff_plan.setAimedCompleteness(aimed_completness)
if char_params.use_aimed_multiplicity:
diff_plan.setAimedMultiplicity(aimed_multiplicity)
if char_params.use_aimed_resolution:
diff_plan.setAimedResolution(aimed_resolution)
diff_plan.setComplexity(complexity)
if char_params.use_permitted_rotation:
diff_plan.setUserDefinedRotationStart(permitted_phi_start)
diff_plan.setUserDefinedRotationRange(rotation_range)
# Vertical crystal dimension
sample = edna_input.getSample()
sample.getSize().setY(XSDataLength(char_params.max_crystal_vdim))
sample.getSize().setZ(XSDataLength(char_params.min_crystal_vdim))
# Radiation damage model
sample.setSusceptibility(XSDataDouble(char_params.rad_suscept))
sample.setChemicalComposition(None)
sample.setRadiationDamageModelBeta(XSDataDouble(char_params.beta /
1e6))
sample.setRadiationDamageModelGamma(
XSDataDouble(char_params.gamma / 1e6))
diff_plan.setForcedSpaceGroup(XSDataString(char_params.space_group))
# Characterisation type - Routine DC
if char_params.use_min_dose:
pass
if char_params.use_min_time:
time = XSDataTime(char_params.min_time)
diff_plan.setMaxExposureTimePerDataCollection(time)
# Account for radiation damage
if char_params.induce_burn:
self._modify_strategy_option(diff_plan, "-DamPar")
# Characterisation type - SAD
if char_params.opt_sad:
if char_params.auto_res:
diff_plan.setAnomalousData(XSDataBoolean(True))
else:
diff_plan.setAnomalousData(XSDataBoolean(False))
self._modify_strategy_option(diff_plan, "-SAD yes")
diff_plan.setAimedResolution(XSDataDouble(char_params.sad_res))
else:
diff_plan.setAnomalousData(XSDataBoolean(False))
# Data set
data_set = XSDataMXCuBEDataSet()
acquisition_parameters = data_collection.acquisitions[
0].acquisition_parameters
path_template = data_collection.acquisitions[0].path_template
path_str = os.path.join(path_template.directory,
path_template.get_image_file_name())
for img_num in range(int(acquisition_parameters.num_images)):
image_file = XSDataFile()
path = XSDataString()
path.setValue(path_str % (img_num + 1))
image_file.setPath(path)
data_set.addImageFile(image_file)
edna_input.addDataSet(data_set)
edna_input.process_directory = path_template.process_directory
return edna_input
def process(self, _edObject=None):
EDPluginExec.process(self)
self.DEBUG("EDPluginExecReadImageHeaderEiger4Mv10.process")
xsDataInputReadImageHeader = self.getDataInput()
xsDataFile = xsDataInputReadImageHeader.getImage()
strPath = xsDataFile.getPath().getValue()
dictEiger4MHeader = self.readHeaderEiger4M(strPath)
if (dictEiger4MHeader is None):
strErrorMessage = "EDPluginExecReadImageHeaderEiger4Mv10.process : Cannot read header : %s" % strPath
self.error(strErrorMessage)
self.addErrorMessage(strErrorMessage)
self.setFailure()
else:
xsDataExperimentalCondition = XSDataExperimentalCondition()
xsDataDetector = XSDataDetector()
iNoPixelsX = 2070
iNoPixelsY = 2167
xsDataDetector.setNumberPixelX(XSDataInteger(iNoPixelsX))
xsDataDetector.setNumberPixelY(XSDataInteger(iNoPixelsY))
# Pixel size
listPixelSizeXY = dictEiger4MHeader["Pixel_size"].split(" ")
fPixelSizeX = float(listPixelSizeXY[0]) * 1000
xsDataDetector.setPixelSizeX(XSDataLength(fPixelSizeX))
fPixelSizeY = float(listPixelSizeXY[3]) * 1000
xsDataDetector.setPixelSizeY(XSDataLength(fPixelSizeY))
# Beam position
listBeamPosition = dictEiger4MHeader["Beam_xy"].replace(
"(", " ").replace(")", " ").replace(",", " ").split()
fBeamPositionX = float(listBeamPosition[1]) * fPixelSizeX
fBeamPositionY = float(listBeamPosition[0]) * fPixelSizeY
xsDataDetector.setBeamPositionX(XSDataLength(fBeamPositionX))
xsDataDetector.setBeamPositionY(XSDataLength(fBeamPositionY))
fDistance = float(
dictEiger4MHeader["Detector_distance"].split(" ")[0]) * 1000
xsDataDetector.setDistance(XSDataLength(fDistance))
# xsDataDetector.setNumberBytesInHeader(XSDataInteger(float(dictEiger4MHeader[ "header_size" ])))
xsDataDetector.setSerialNumber(
XSDataString(dictEiger4MHeader["Detector:"]))
# #xsDataDetector.setBin( XSDataString( dictEiger4MHeader[ "BIN" ] ) ) )
# #xsDataDetector.setDataType( XSDataString( dictEiger4MHeader[ "TYPE" ] ) ) )
# #xsDataDetector.setByteOrder( XSDataString( dictEiger4MHeader[ "BYTE_ORDER" ] ) ) )
# xsDataDetector.setImageSaturation(XSDataInteger(int(dictEiger4MHeader[ "saturation_level" ])))
xsDataDetector.setName(XSDataString("EIGER 4M"))
xsDataDetector.setType(XSDataString("eiger4m"))
xsDataExperimentalCondition.setDetector(xsDataDetector)
# Beam object
xsDataBeam = XSDataBeam()
xsDataBeam.setWavelength(
XSDataWavelength(
float(dictEiger4MHeader["Wavelength"].split(" ")[0])))
xsDataBeam.setExposureTime(
XSDataTime(
float(dictEiger4MHeader["Exposure_time"].split(" ")[0])))
xsDataExperimentalCondition.setBeam(xsDataBeam)
# Goniostat object
xsDataGoniostat = XSDataGoniostat()
fRotationAxisStart = float(
dictEiger4MHeader["Start_angle"].split(" ")[0])
fOscillationWidth = float(
dictEiger4MHeader["Angle_increment"].split(" ")[0])
xsDataGoniostat.setRotationAxisStart(
XSDataAngle(fRotationAxisStart))
xsDataGoniostat.setRotationAxisEnd(
XSDataAngle(fRotationAxisStart + fOscillationWidth))
xsDataGoniostat.setOscillationWidth(XSDataAngle(fOscillationWidth))
xsDataExperimentalCondition.setGoniostat(xsDataGoniostat)
#
# Create the image object
xsDataImage = XSDataImage()
xsDataImage.setPath(XSDataString(strPath))
if "DateTime" in dictEiger4MHeader:
strTimeStamp = dictEiger4MHeader["DateTime"]
xsDataImage.setDate(XSDataString(strTimeStamp))
iImageNumber = EDUtilsImage.getImageNumber(strPath)
xsDataImage.setNumber(XSDataInteger(iImageNumber))
xsDataSubWedge = XSDataSubWedge()
xsDataSubWedge.setExperimentalCondition(
xsDataExperimentalCondition)
xsDataSubWedge.addImage(xsDataImage)
self.__xsDataResultReadImageHeader = XSDataResultReadImageHeader()
self.__xsDataResultReadImageHeader.setSubWedge(xsDataSubWedge)
def process(self, _edObject=None):
EDPlugin.process(self)
output = XSDataXdsOutput()
# get all the file's contents, find where the info is and then
# use helper functions to retrieve stuff and put it in the
# data model
try:
f = open(self.dataInput.correct_lp.path.value, 'r')
lines = f.readlines()
except IOError:
EDVerbose.ERROR(
'Could not open the specified XDS output file for reading')
self.setFailure()
return
# look for the "big piece of information"
info_begin = None
info_end = None
for lineno, line in enumerate(lines):
if info_begin is None:
if line.find(
'REFINEMENT OF DIFFRACTION PARAMETERS USING ALL IMAGES'
) != -1:
info_begin = lineno
else:
if line.find(
'MEAN INTENSITY AS FUNCTION OF SPINDLE POSITION WITHIN DATA IMAGE'
) != -1:
info_end = lineno
break
if info_begin is None or info_end is None:
EDVerbose.ERROR('could not find the refined parameters')
self.setFailure()
return
_extract_infos(lines[info_begin:info_end], output)
# second pass, look for the interesting table
info_begin = None
info_end = None
started = False
for line_no, line in enumerate(lines):
if line.find(
'REFLECTIONS OF TYPE H,0,0 0,K,0 0,0,L OR EXPECTED TO BE ABSENT (*)'
) != -1:
# the table will start shortly after
started = True
continue
if started:
# look if we are at the table yet
if line.find(
'LIMIT OBSERVED UNIQUE POSSIBLE OF DATA observed expected'
) != -1:
# there's an empty line after the header
info_begin = line_no + 2
if info_begin is not None and line.find('total') != -1:
# we're at the last table line
info_end = line_no
if info_begin is None or info_end is None:
EDVerbose.ERROR('could not find the completeness table')
self.setFailure()
return
_extract_completeness_entries(lines[info_begin:info_end + 1], output)
# now for the last bit: check if we were given a path to the
# gxparm file and if it exists get the space group and unit
# cell constants from it
if self.dataInput.gxparm is not None:
gxparm_path = self.dataInput.gxparm.path.value
if os.path.exists(gxparm_path):
with open(gxparm_path, 'r') as f:
lines = f.readlines()
for line in lines:
# the one we want has 7 floats
chunks = line.split()
if len(chunks) == 7:
output.sg_number = XSDataInteger(int(chunks[0]))
output.unit_cell_constants = [
XSDataFloat(float(x)) for x in chunks[1:]
]
input_file = self.dataInput.correct_lp.path.value
output.xds_run_directory = XSDataString(os.path.dirname(input_file))
self.dataOutput = output
def postProcess(self, _edObject=None):
"""
complex type XSDataBioSaxsExperimentSetup extends XSData{
detector : XSDataString optional
detectorDistance : XSDataLength optional
pixelSize_1 : XSDataLength optional
pixelSize_2 : XSDataLength optional
beamCenter_1 : XSDataDouble optional
beamCenter_2 : XSDataDouble optional
beamStopDiode : XSDataDouble optional
wavelength : XSDataWavelength optional
machineCurrent : XSDataDouble optional
maskFile : XSDataImage optional
normalizationFactor : XSDataDouble optional
storageTemperature: XSDataDouble optional
exposureTemperature: XSDataDouble optional
exposureTime: XSDataTime optional
frameNumber: XSDataInteger optional
frameMax: XSDataInteger optional
}
complex type XSDataBioSaxsSample extends XSData {
concentration : XSDataDouble optional
comments : XSDataString optional
code : XSDataString optional
temperature: XSDataDouble optional
"""
EDPluginExec.postProcess(self)
EDVerbose.DEBUG("EDPluginBioSaxsMetadatav1_1.postProcess")
# Create some output data
xsDataResult = XSDataResultBioSaxsMetadatav1_0()
xsdSample = XSDataBioSaxsSample()
xsdExperiment = XSDataBioSaxsExperimentSetup()
if self.strOutputImage is not None:
xsdImage = XSDataImage()
xsdImage.setPath(XSDataString(self.strOutputImage))
xsDataResult.setOutputImage(xsdImage)
if self.detector is not None:
xsdExperiment.detector = xsDataResult.detector = XSDataString(
self.detector)
if self.detectorDistance is not None:
xsdExperiment.detectorDistance = xsDataResult.detectorDistance = XSDataLength(
self.detectorDistance)
if self.pixelSize_1 is not None:
xsDataResult.pixelSize_1 = XSDataLength(self.pixelSize_1)
xsdExperiment.pixelSize_1 = xsDataResult.pixelSize_1
if self.pixelSize_2 is not None:
xsDataResult.pixelSize_2 = XSDataLength(self.pixelSize_2)
xsdExperiment.pixelSize_2 = xsDataResult.pixelSize_2
if self.beamCenter_1 is not None:
xsDataResult.beamCenter_1 = XSDataDouble(self.beamCenter_1)
xsdExperiment.beamCenter_1 = xsDataResult.beamCenter_1
if self.beamCenter_2 is not None:
xsDataResult.beamCenter_2 = XSDataDouble(self.beamCenter_2)
xsdExperiment.beamCenter_2 = xsDataResult.beamCenter_2
if self.beamStopDiode is not None:
xsdExperiment.beamStopDiode = xsDataResult.beamStopDiode = XSDataDouble(
self.beamStopDiode)
if self.wavelength is not None:
xsdExperiment.wavelength = xsDataResult.wavelength = XSDataWavelength(
self.wavelength)
if self.maskFile is not None:
xsdFile = XSDataImage()
xsdFile.setPath(XSDataString(self.maskFile))
xsdExperiment.maskFile = xsDataResult.maskFile = xsdFile
if self.normalizationFactor is not None:
xsdExperiment.normalizationFactor = xsDataResult.normalizationFactor = XSDataDouble(
self.normalizationFactor)
if self.machineCurrent is not None:
xsdExperiment.machineCurrent = xsDataResult.machineCurrent = XSDataDouble(
self.machineCurrent)
if self.storageTemperature is not None:
xsdExperiment.storageTemperature = xsDataResult.storageTemperature = XSDataDouble(
self.storageTemperature)
if self.exposureTemperature is not None:
xsdExperiment.exposureTemperature = xsDataResult.exposureTemperature = XSDataDouble(
self.exposureTemperature)
if self.exposureTime is not None:
xsdExperiment.exposureTime = xsDataResult.exposureTime = XSDataTime(
self.exposureTime)
if self.frameNumber is not None:
xsdExperiment.frameNumber = xsDataResult.frameNumber = XSDataInteger(
self.frameNumber)
if self.frameMax is not None:
xsdExperiment.frameMax = xsDataResult.frameMax = XSDataInteger(
self.frameMax)
if self.timeOfFrame is not None:
xsdExperiment.timeOfFrame = xsDataResult.timeOfFrame = XSDataTime(
self.timeOfFrame)
if self.code is not None:
xsdSample.code = xsDataResult.code = XSDataString(self.code)
if self.comments is not None:
xsdSample.comments = xsDataResult.comments = XSDataString(
self.comments)
if self.concentration is not None:
xsdSample.concentration = xsDataResult.concentration = XSDataDouble(
self.concentration)
xsDataResult.sample = xsdSample
xsDataResult.experimentSetup = xsdExperiment
self.setDataOutput(xsDataResult)
def finallyProcess(self, _edObject=None):
EDPluginISPyBv1_4.finallyProcess(self)
self.DEBUG("EDPluginISPyBUpdateDataCollectionGroupWorkflowIdv1_4.finallyProcess")
xsDataResultISPyBUpdateDataCollectionGroupWorkflowId = XSDataResultISPyBUpdateDataCollectionGroupWorkflowId()
if self.iDataCollectionGroupId is not None:
xsDataResultISPyBUpdateDataCollectionGroupWorkflowId.dataCollectionGroupId = XSDataInteger(self.iDataCollectionGroupId)
self.setDataOutput(xsDataResultISPyBUpdateDataCollectionGroupWorkflowId)
def process(self, _edObject=None):
"""
Sends a store request and the screening object to the dbserver. Returns success or failure.
Note that:
* Any objects referred to by the object returned by self.getDataInput() will be stored.
* Primary key attributes should not be set.
* If a foreign key attribute is not set, this method will attempt to find the foreign object among those
referred to by the self.getDataInput() object, and use the primary key attribute of this object when it
has been stored. If such an object is not found, the method fails.
* The method will not attempt to store any more objects once an error is encountered
"""
EDPluginExec.process(self)
self.DEBUG("*** EDPluginISPyBv10.process")
xsDataInputISPyB = self.getDataInput()
self.m_xsDataResultISPyB = XSDataResultISPyB()
xsDataISPyBScreening = xsDataInputISPyB.getScreening()
if xsDataISPyBScreening == None:
xsDataISPyBScreening = XSDataISPyBScreening()
xsDataISPyBImage = xsDataInputISPyB.getImage()
if xsDataISPyBScreening.getDataCollectionId() == None:
if ((xsDataISPyBImage == None)
or (xsDataISPyBImage.getFileName() == None)
or (xsDataISPyBImage.getFileLocation() == None)):
pyStrErrorMessage = EDMessage.ERROR_EXECUTION_03 % ("EDPluginISPyBv10", "process", \
"Neither a dataCollectionId nor a image filename + path are provided.")
self.error(pyStrErrorMessage)
self.addErrorMessage(pyStrErrorMessage)
return
strXML = EDUtilsXML.dnaMarshal(xsDataISPyBImage)
# Send the XML to request the dataCollectionId from the dbserver:
edStringResponse = self.httpPost(self.getDbserverHost(),
self.getDbserverPort(),
"/get_datacollectionid", strXML)
if edStringResponse != None:
# Handle response:
self.DEBUG(edStringResponse)
xsDatadbstatus = XSDatadbstatus.parseString(edStringResponse)
pyStrCode = xsDatadbstatus.getCode()
pyStrMessage = xsDatadbstatus.getMessage()
self.DEBUG("dbserver returns code: " + pyStrCode)
self.DEBUG("dbserver returns message: " + pyStrMessage)
if (pyStrCode
== "error") or (xsDatadbstatus.getDataCollectionId()
== -1):
if xsDatadbstatus.getDataCollectionId() == -1:
pyStrMessage = "An image corresponding to the given fileName and fileLocation was not found."
pyStrErrorMessage = EDMessage.ERROR_EXECUTION_03 % (
"EDPluginISPyBv10", "process", pyStrMessage)
self.error(pyStrErrorMessage)
self.addErrorMessage(pyStrErrorMessage)
return
if xsDatadbstatus.getDataCollectionId() != -1:
xsDataISPyBScreening.setDataCollectionId(
XSDataInteger(xsDatadbstatus.getDataCollectionId()))
self.DEBUG("dataCollectionId is: " + str(
xsDataISPyBScreening.getDataCollectionId().getValue()))
xsDataISPyBScreeningInput = xsDataInputISPyB.getScreeningInput()
xsDataISPyBScreeningOutput = xsDataInputISPyB.getScreeningOutput()
xsDataISPyBScreeningOutputLattice = xsDataInputISPyB.getScreeningOutputLattice(
)
xsDataISPyBScreeningRank = xsDataInputISPyB.getScreeningRank()
xsDataISPyBScreeningRankSet = xsDataInputISPyB.getScreeningRankSet()
xsDataISPyBScreeningStrategy = xsDataInputISPyB.getScreeningStrategy()
for xsDataISPyBScreeningObject in [ xsDataISPyBScreening, xsDataISPyBScreeningRankSet, xsDataISPyBScreeningInput, xsDataISPyBScreeningOutput, \
xsDataISPyBScreeningOutputLattice, xsDataISPyBScreeningRank, xsDataISPyBScreeningStrategy ]:
if xsDataISPyBScreeningObject != None:
# Set any missing foreign key attributes in xsDataScreeningObject:
self.setForeignKeyAttributes(xsDataISPyBScreeningObject,
self.m_xsDataResultISPyB)
# Create DNA compatible XML from xsDataScreeningObject:
strXML = EDUtilsXML.dnaMarshal(xsDataISPyBScreeningObject)
# Send the XML to the dbserver:
edStringResponse = self.httpPost(self.getDbserverHost(),
self.getDbserverPort(),
"/store_object_request",
strXML)
if edStringResponse != None:
# Handle response:
self.DEBUG(edStringResponse)
xsDatadbstatus = XSDatadbstatus.parseString(
edStringResponse)
pyStrCode = xsDatadbstatus.getCode()
pyStrMessage = xsDatadbstatus.getMessage()
xsDataStrCode = XSDataString(pyStrCode)
xsDataStrMessage = XSDataString(pyStrMessage)
self.DEBUG("dbserver returns code: " + pyStrCode)
self.DEBUG("dbserver returns message: " + pyStrMessage)
xsDataResultStatusList = self.m_xsDataResultISPyB.getResultStatus(
)
xsDataResultStatus = XSDataResultStatus(
xsDataISPyBScreeningObject, xsDataStrCode,
xsDataStrMessage)
xsDataResultStatusList.append(xsDataResultStatus)
self.addToXSDataResultISPyB(self.m_xsDataResultISPyB,
xsDatadbstatus)
if pyStrCode == "error":
pyStrErrorMessage = EDMessage.ERROR_EXECUTION_03 % (
"EDPluginISPyBv10", "process", pyStrMessage)
self.error(pyStrErrorMessage)
self.addErrorMessage(pyStrErrorMessage)
return
else:
return
def parseLabelitDistlOutput(self, _strLabelitDistlLogText):
self.DEBUG("EDPluginDistlSignalStrengthv1_1.parseLabelitDistlOutput")
xsDataImageQualityIndicators = None
if _strLabelitDistlLogText is not None:
xsDataImageQualityIndicators = XSDataImageQualityIndicators()
for strLine in _strLabelitDistlLogText.split("\n"):
if strLine.find("Spot Total") != -1:
iSpotTotal = int(strLine.split()[3])
xsDataImageQualityIndicators.setSpotTotal(
XSDataInteger(iSpotTotal))
elif strLine.find("In-Resolution Total") != -1:
iInResTotal = int(strLine.split()[3])
xsDataImageQualityIndicators.setInResTotal(
XSDataInteger(iInResTotal))
elif strLine.find("Good Bragg Candidates") != -1:
iGoodBraggCandidates = int(strLine.split()[4])
xsDataImageQualityIndicators.setGoodBraggCandidates(
XSDataInteger(iGoodBraggCandidates))
elif strLine.find("Ice Rings") != -1:
iIceRings = int(strLine.split()[3])
xsDataImageQualityIndicators.setIceRings(
XSDataInteger(iIceRings))
elif strLine.find("Method 1 Resolution") != -1:
fMethod1Res = float(strLine.split()[4])
xsDataImageQualityIndicators.setMethod1Res(
XSDataDouble(fMethod1Res))
elif strLine.find("Method 2 Resolution") != -1:
if strLine.split()[4] != "None":
fMethod2Res = float(strLine.split()[4])
xsDataImageQualityIndicators.setMethod2Res(
XSDataDouble(fMethod2Res))
elif strLine.find("Maximum unit cell") != -1:
if strLine.split()[4] != "None":
fMaxUnitCell = float(strLine.split()[4])
xsDataImageQualityIndicators.setMaxUnitCell(
XSDataDouble(fMaxUnitCell))
elif strLine.find("%Saturation, Top 50 Peaks") != -1:
fPctSaturationTop50Peaks = float(strLine.split()[5])
xsDataImageQualityIndicators.setPctSaturationTop50Peaks(
XSDataDouble(fPctSaturationTop50Peaks))
elif strLine.find("In-Resolution Ovrld Spots") != -1:
iInResolutionOvrlSpots = int(strLine.split()[4])
xsDataImageQualityIndicators.setInResolutionOvrlSpots(
XSDataInteger(iInResolutionOvrlSpots))
elif strLine.find(
"Bin population cutoff for method 2 resolution") != -1:
fBinPopCutOffMethod2Res = float(strLine.split()[7][:-1])
xsDataImageQualityIndicators.setBinPopCutOffMethod2Res(
XSDataDouble(fBinPopCutOffMethod2Res))
elif strLine.find(
"Total integrated signal, pixel-ADC units above local background (just the good Bragg candidates)"
) != -1:
fTotalIntegratedSignal = float(strLine.split()[-1])
xsDataImageQualityIndicators.setTotalIntegratedSignal(
XSDataDouble(fTotalIntegratedSignal))
elif strLine.find("Signals range from") != -1:
listStrLine = strLine.split()
fSignalRangeMin = float(listStrLine[3])
fSignalRangeMax = float(listStrLine[5])
fSignalRangeAverage = float(listStrLine[-1])
xsDataImageQualityIndicators.setSignalRangeMin(
XSDataDouble(fSignalRangeMin))
xsDataImageQualityIndicators.setSignalRangeMax(
XSDataDouble(fSignalRangeMax))
xsDataImageQualityIndicators.setSignalRangeAverage(
XSDataDouble(fSignalRangeAverage))
elif strLine.find("Saturations range from") != -1:
listStrLine = strLine.split()
fSaturationRangeMin = float(listStrLine[3][:-1])
fSaturationRangeMax = float(listStrLine[5][:-1])
fSaturationRangeAverage = float(listStrLine[-1][:-1])
xsDataImageQualityIndicators.setSaturationRangeMin(
XSDataDouble(fSaturationRangeMin))
xsDataImageQualityIndicators.setSaturationRangeMax(
XSDataDouble(fSaturationRangeMax))
xsDataImageQualityIndicators.setSaturationRangeAverage(
XSDataDouble(fSaturationRangeAverage))
return xsDataImageQualityIndicators
def testSetDataModelInput(self):
"""
"""
edPluginStrategy = self.createPlugin()
xsPluginItemGood01 = self.getPluginConfiguration(os.path.join(self.strDataPath, "XSConfiguration_ESRF.xml"))
edPluginStrategy.setConfiguration(xsPluginItemGood01)
edPluginStrategy.configure()
from XSDataMXv1 import XSDataStrategyInput
xSDataStrategy = XSDataStrategyInput()
# Beam
from XSDataCommon import XSDataFlux
from XSDataCommon import XSDataWavelength
from XSDataCommon import XSDataSize
from XSDataCommon import XSDataLength
from XSDataCommon import XSDataTime
from XSDataMXv1 import XSDataBeam
from XSDataMXv1 import XSDataExperimentalCondition
xsExperimentalCondition = XSDataExperimentalCondition()
xsBeam = XSDataBeam()
xsBeam.setFlux(XSDataFlux(1e+12))
xsBeam.setWavelength(XSDataWavelength(2.41))
xsBeam.setSize(XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1)))
xsBeam.setExposureTime(XSDataTime(1))
xsExperimentalCondition.setBeam(xsBeam)
# Detector and Exposure Time
from XSDataMXv1 import XSDataDetector
from XSDataCommon import XSDataString
from XSDataMXv1 import XSDataGoniostat
xsDataDetector = XSDataDetector()
xsDataDetector.setType(XSDataString("q210-2x"))
xsExperimentalCondition.setDetector(xsDataDetector)
xsDataGoniostat = XSDataGoniostat()
xsDataGoniostat.setRotationAxis(XSDataString("phi"))
xsExperimentalCondition.setGoniostat(xsDataGoniostat)
xSDataStrategy.setExperimentalCondition(xsExperimentalCondition)
# Best Files
bestFileContentDat = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile.dat"))
xSDataStrategy.setBestFileContentDat(XSDataString(bestFileContentDat))
bestFileContentPar = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile.par"))
xSDataStrategy.setBestFileContentPar(XSDataString(bestFileContentPar))
bestFileContentHKL = EDUtilsFile.readFile(os.path.join(self.strDataPath, "bestfile1.hkl"))
listBestFileContentHKL = []
listBestFileContentHKL.append(XSDataString(bestFileContentHKL))
xSDataStrategy.setBestFileContentHKL(listBestFileContentHKL)
# Crystal
from XSDataCommon import XSDataFloat
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataInteger
from XSDataMXv1 import XSDataCrystal
from XSDataMXv1 import XSDataStructure
from XSDataMXv1 import XSDataChain
from XSDataMXv1 import XSDataAtom
from XSDataMXv1 import XSDataLigand
from XSDataMXv1 import XSDataSampleCrystalMM
from XSDataMXv1 import XSDataChemicalCompositionMM
from XSDataMXv1 import XSDataAtomicComposition
from XSDataMXv1 import XSDataSolvent
from XSDataMXv1 import XSDataCell
from XSDataMXv1 import XSDataSpaceGroup
xsDataSampleCrystalMM = XSDataSampleCrystalMM()
xsDataStructure = XSDataStructure()
xsDataComposition = XSDataChemicalCompositionMM()
xsDataChain = XSDataChain()
xsDataChain.setType(XSDataString("protein"))
xsDataChain.setNumberOfCopies(XSDataFloat(2))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom1 = XSDataAtom()
xsDataAtom1.setSymbol(XSDataString("Se"))
xsDataAtom1.setNumberOf(XSDataFloat(4))
xsDataAtomicComposition.addAtom(xsDataAtom1)
xsDataChain.setHeavyAtoms(xsDataAtomicComposition)
xsDataChain.setNumberOfMonomers(XSDataFloat(100))
xsDataStructure.addChain(xsDataChain)
xsDataChain2 = XSDataChain()
xsDataChain2.setType(XSDataString("rna"))
xsDataChain2.setNumberOfCopies(XSDataFloat(1))
xsDataChain2.setNumberOfMonomers(XSDataFloat(60))
xsDataStructure.addChain(xsDataChain2)
xsDataLigand = XSDataLigand()
xsDataLigand.setNumberOfCopies(XSDataFloat(2))
xsDataLigand.setNumberOfLightAtoms(XSDataFloat(42))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom2 = XSDataAtom()
xsDataAtom2.setSymbol(XSDataString("Fe"))
xsDataAtom2.setNumberOf(XSDataFloat(1))
xsDataAtomicComposition.addAtom(xsDataAtom2)
xsDataLigand.setHeavyAtoms(xsDataAtomicComposition)
xsDataStructure.addLigand(xsDataLigand)
xsDataStructure.setNumberOfCopiesInAsymmetricUnit(XSDataFloat(0.25))
xsDataSolvent = XSDataSolvent()
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtom3 = XSDataAtom()
xsDataAtom3.setSymbol(XSDataString("Na"))
xsDataAtom3.setConcentration(XSDataFloat(1000))
xsDataAtom4 = XSDataAtom()
xsDataAtom4.setSymbol(XSDataString("Cl"))
xsDataAtom4.setConcentration(XSDataFloat(1000))
xsDataAtomicComposition.addAtom(xsDataAtom3)
xsDataAtomicComposition.addAtom(xsDataAtom4)
xsDataSolvent.setAtoms(xsDataAtomicComposition)
xsDataComposition.setStructure(xsDataStructure)
xsDataComposition.setSolvent(xsDataSolvent)
xsDataSampleCrystalMM.setChemicalComposition(xsDataComposition)
xsDataSampleCrystalMM.setSize(XSDataSize(XSDataLength(0.1), XSDataLength(0.1), XSDataLength(0.1)))
xsDataCrystal = XSDataCrystal()
xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0),
angle_beta=XSDataAngle(90.0),
angle_gamma=XSDataAngle(90.0),
length_a=XSDataLength(78.9),
length_b=XSDataLength(95.162),
length_c=XSDataLength(104.087))
xsDataCrystal.setCell(xsDataCell)
xsDataSpaceGroup = XSDataSpaceGroup()
xsDataSpaceGroup.setITNumber(XSDataInteger(16))
xsDataCrystal.setSpaceGroup(xsDataSpaceGroup)
xsDataSampleCrystalMM.setSusceptibility(XSDataFloat(1.5))
xSDataStrategy.setCrystalRefined(xsDataCrystal)
xSDataStrategy.setSample(xsDataSampleCrystalMM)
xSDataStrategy.outputFile(self.strObtainedInputFile)
strExpectedInput = self.readAndParseFile (self.strReferenceInputFile)
strObtainedInput = self.readAndParseFile (self.strObtainedInputFile)
xsDataInputExpected = XSDataStrategyInput.parseString(strExpectedInput)
xsDataInputObtained = XSDataStrategyInput.parseString(strObtainedInput)
EDAssert.equal(xsDataInputExpected.marshal(), xsDataInputObtained.marshal())
def postProcess(self, _edObject=None):
EDPluginControl.postProcess(self)
self.DEBUG("EDPluginControlDozorv1_0.postProcess")
# Write a file to be used with ISPyB or GNUPLOT only if data collection id in input
dataCollectionId = None
if self.dataInput.dataCollectionId is None and len(self.dataInput.image) > 0:
# Only try to obtain data collection id if at the ESRF and path starts with "/data"
if EDUtilsPath.isESRF() and self.dataInput.image[0].path.value.startswith("/data"):
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.image = XSDataImage(self.dataInput.image[0].path)
edPluginISPyBRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
edPluginISPyBRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginISPyBRetrieveDataCollection.executeSynchronous()
xsDataResultRetrieveDataCollection = edPluginISPyBRetrieveDataCollection.dataOutput
if xsDataResultRetrieveDataCollection is not None:
dataCollection = xsDataResultRetrieveDataCollection.dataCollection
if dataCollection is not None:
dataCollectionId = dataCollection.dataCollectionId
elif self.dataInput.dataCollectionId is not None:
dataCollectionId = self.dataInput.dataCollectionId.value
if dataCollectionId is not None:
minImageNumber = None
maxImageNumber = None
minAngle = None
maxAngle = None
minDozorValue = None
maxDozorValue = None
minResolution = None
maxResolution = None
dozorPlotFileName = "dozor_{0}.png".format(dataCollectionId)
dozorCsvFileName = "dozor_{0}.csv".format(dataCollectionId)
with open(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), "w") as gnuplotFile:
gnuplotFile.write("# Data directory: {0}\n".format(self.directory))
gnuplotFile.write("# File template: {0}\n".format(self.template.replace("%04d", "####")))
gnuplotFile.write("# {0:>9s}{1:>16s}{2:>16s}{3:>16s}{4:>16s}{5:>16s}\n".format("'Image no'",
"'Angle'",
"'No of spots'",
"'Main score (*10)'",
"'Spot score'",
"'Visible res.'",
))
for imageDozor in self.dataOutput.imageDozor:
gnuplotFile.write("{0:10d},{1:15.3f},{2:15d},{3:15.3f},{4:15.3f},{5:15.3f}\n".format(imageDozor.number.value,
imageDozor.angle.value,
imageDozor.spotsNumOf.value,
10 * imageDozor.mainScore.value,
imageDozor.spotScore.value,
imageDozor.visibleResolution.value,
))
if minImageNumber is None or minImageNumber > imageDozor.number.value:
minImageNumber = imageDozor.number.value
minAngle = imageDozor.angle.value
if maxImageNumber is None or maxImageNumber < imageDozor.number.value:
maxImageNumber = imageDozor.number.value
maxAngle = imageDozor.angle.value
if minDozorValue is None or minDozorValue > imageDozor.mainScore.value:
minDozorValue = imageDozor.spotScore.value
if maxDozorValue is None or maxDozorValue < imageDozor.mainScore.value:
maxDozorValue = imageDozor.spotScore.value
# Min resolution: the higher the value the lower the resolution
if minResolution is None or minResolution < imageDozor.visibleResolution.value:
# Disregard resolution worse than 10.0
if imageDozor.visibleResolution.value < 10.0:
minResolution = imageDozor.visibleResolution.value
# Max resolution: the lower the number the better the resolution
if maxResolution is None or maxResolution > imageDozor.visibleResolution.value:
maxResolution = imageDozor.visibleResolution.value
xtics = ""
if minImageNumber is not None and minImageNumber == maxImageNumber:
minAngle -= 1.0
maxAngle += 1.0
noImages = maxImageNumber - minImageNumber + 1
if noImages <= 4:
minImageNumber -= 0.1
maxImageNumber += 0.1
deltaAngle = maxAngle - minAngle
minAngle -= deltaAngle * 0.1 / noImages
maxAngle += deltaAngle * 0.1 / noImages
xtics = "1"
if maxResolution is None or maxResolution > 0.8:
maxResolution = 0.8
else:
maxResolution = int(maxResolution * 10.0) / 10.0
if minResolution is None or minResolution < 4.5:
minResolution = 4.5
else:
minResolution = int(minResolution * 10.0) / 10.0 + 1
if maxDozorValue < 0.001 and minDozorValue < 0.001:
yscale = "set yrange [-0.5:0.5]\n set ytics 1"
else:
yscale = "set autoscale y"
gnuplotFile.close()
gnuplotScript = \
"""#
set terminal png
set output '{dozorPlotFileName}'
set title '{title}'
set grid x2 y2
set xlabel 'Image number'
set x2label 'Angle (degrees)'
set y2label 'Resolution (A)'
set ylabel 'Number of spots / Dozor score (*10)'
set xtics {xtics} nomirror
set x2tics
set ytics nomirror
set y2tics
set xrange [{minImageNumber}:{maxImageNumber}]
set x2range [{minAngle}:{maxAngle}]
{yscale}
set y2range [{minResolution}:{maxResolution}]
set key below
plot '{dozorCsvFileName}' using 1:3 title 'Number of spots' axes x1y1 with points linetype rgb 'goldenrod' pointtype 7 pointsize 1.5, \
'{dozorCsvFileName}' using 1:4 title 'Dozor score' axes x1y1 with points linetype 3 pointtype 7 pointsize 1.5, \
'{dozorCsvFileName}' using 1:6 title 'Visible resolution' axes x1y2 with points linetype 1 pointtype 7 pointsize 1.5
""".format(title=self.template.replace("%04d", "####"),
dozorPlotFileName=dozorPlotFileName,
dozorCsvFileName=dozorCsvFileName,
minImageNumber=minImageNumber,
maxImageNumber=maxImageNumber,
minAngle=minAngle,
maxAngle=maxAngle,
minResolution=minResolution,
maxResolution=maxResolution,
xtics=xtics,
yscale=yscale,
)
pathGnuplotScript = os.path.join(self.getWorkingDirectory(), "gnuplot.sh")
data_file = open(pathGnuplotScript, "w")
data_file.write(gnuplotScript)
data_file.close()
oldCwd = os.getcwd()
os.chdir(self.getWorkingDirectory())
os.system("{0} {1}".format(self.gnuplot, pathGnuplotScript))
os.chdir(oldCwd)
self.dataOutput.dozorPlot = XSDataFile(XSDataString(os.path.join(self.getWorkingDirectory(), dozorPlotFileName)))
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = os.path.join(self.directory.replace("RAW_DATA", "PROCESSED_DATA"), "DozorPlot")
resultsDirectory = os.path.join(processDirectory, "results")
dozorPlotResultPath = os.path.join(resultsDirectory, dozorPlotFileName)
dozorCsvResultPath = os.path.join(resultsDirectory, dozorCsvFileName)
try:
if not os.path.exists(resultsDirectory):
os.makedirs(resultsDirectory, 0o755)
shutil.copy(os.path.join(self.getWorkingDirectory(), dozorPlotFileName), dozorPlotResultPath)
shutil.copy(os.path.join(self.getWorkingDirectory(), dozorCsvFileName), dozorCsvResultPath)
except:
self.warning("Couldn't copy files to results directory: {0}".format(resultsDirectory))
try:
# Create paths on pyarch
dozorPlotPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorPlotResultPath)
dozorCsvPyarchPath = EDHandlerESRFPyarchv1_0.createPyarchFilePath(dozorCsvResultPath)
if not os.path.exists(os.path.dirname(dozorPlotPyarchPath)):
os.makedirs(os.path.dirname(dozorPlotPyarchPath), 0o755)
shutil.copy(dozorPlotResultPath, dozorPlotPyarchPath)
shutil.copy(dozorCsvResultPath, dozorCsvPyarchPath)
# Upload to data collection
xsDataInputISPyBSetImageQualityIndicatorsPlot = XSDataInputISPyBSetImageQualityIndicatorsPlot()
xsDataInputISPyBSetImageQualityIndicatorsPlot.dataCollectionId = XSDataInteger(dataCollectionId)
xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsPlotPath = XSDataString(dozorPlotPyarchPath)
xsDataInputISPyBSetImageQualityIndicatorsPlot.imageQualityIndicatorsCSVPath = XSDataString(dozorCsvPyarchPath)
EDPluginISPyBSetImageQualityIndicatorsPlot = self.loadPlugin("EDPluginISPyBSetImageQualityIndicatorsPlotv1_4")
EDPluginISPyBSetImageQualityIndicatorsPlot.dataInput = xsDataInputISPyBSetImageQualityIndicatorsPlot
EDPluginISPyBSetImageQualityIndicatorsPlot.executeSynchronous()
except:
self.warning("Couldn't copy files to pyarch: {0}".format(dozorPlotPyarchPath))
self.sendMessageToMXCuBE("Processing finished", "info")
self.setStatusToMXCuBE("Success")
def convertToCBF(self, dictImage, listAllBatches, doRadiationDamage=False):
# Find start and end image number
startImage = None
endImage = None
for image in dictImage:
if startImage is None or startImage > image:
startImage = image
if endImage is None or endImage < image:
endImage = image
# Check if we are dealing with characterisation images
newDict = {}
hasHdf5Prefix = True
if self.hasOverlap or startImage == endImage:
hasHdf5Prefix = False
for image in dictImage:
xsDataInputH5ToCBF = XSDataInputH5ToCBF()
xsDataInputH5ToCBF.hdf5File = dictImage[startImage]
xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(image)
xsDataInputH5ToCBF.imageNumber = XSDataInteger(startImage)
xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(self.cbfTempDir))
xsDataInputH5ToCBF.forcedOutputImageNumber = XSDataInteger(image)
edPluginH5ToCBF = self.loadPlugin("EDPluginH5ToCBFv1_1")
edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
edPluginH5ToCBF.executeSynchronous()
newDict[image] = edPluginH5ToCBF.dataOutput.outputCBFFile
else:
listPluginH5ToCBF = []
directory = os.path.dirname(dictImage[listAllBatches[0][0]].path.value)
for batch in listAllBatches:
xsDataInputH5ToCBF = XSDataInputH5ToCBF()
if doRadiationDamage:
xsDataInputH5ToCBF.hdf5File = dictImage[batch[0]]
xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(batch[0])
xsDataInputH5ToCBF.startImageNumber = XSDataInteger(listAllBatches[0][0])
xsDataInputH5ToCBF.endImageNumber = XSDataInteger(listAllBatches[0][-1])
else:
xsDataInputH5ToCBF.hdf5File = dictImage[startImage]
xsDataInputH5ToCBF.hdf5ImageNumber = XSDataInteger(1)
xsDataInputH5ToCBF.startImageNumber = XSDataInteger(batch[0])
xsDataInputH5ToCBF.endImageNumber = XSDataInteger(batch[-1])
xsDataInputH5ToCBF.forcedOutputDirectory = XSDataFile(XSDataString(self.cbfTempDir))
edPluginH5ToCBF = self.loadPlugin("EDPluginH5ToCBFv1_1")
edPluginH5ToCBF.dataInput = xsDataInputH5ToCBF
if doRadiationDamage:
edPluginH5ToCBF.executeSynchronous()
if edPluginH5ToCBF.dataOutput is not None and edPluginH5ToCBF.dataOutput.outputCBFFileTemplate is not None:
outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate
for newImageNumber in batch:
oldImageNumber = newImageNumber - batch[0] + 1
oldPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:06d}".format(oldImageNumber)))
newPath = os.path.join(directory, outputCBFFileTemplate.path.value.replace("######", "{0:04d}".format(newImageNumber)))
os.rename(oldPath, newPath)
newDict[newImageNumber] = XSDataFile(XSDataString(newPath))
hasHdf5Prefix = False
else:
edPluginH5ToCBF.execute()
listPluginH5ToCBF.append(edPluginH5ToCBF)
for edPluginH5ToCBF in listPluginH5ToCBF:
edPluginH5ToCBF.synchronize()
if edPluginH5ToCBF.dataOutput is not None and edPluginH5ToCBF.dataOutput.outputCBFFileTemplate is not None:
outputCBFFileTemplate = edPluginH5ToCBF.dataOutput.outputCBFFileTemplate.path.value
outputCBFFileTemplate = outputCBFFileTemplate.replace("######", "{0:06d}")
for image in dictImage:
newDict[image] = XSDataFile(XSDataString(outputCBFFileTemplate.format(image)))
return newDict, hasHdf5Prefix
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlDozorv1_0.process")
self.sendMessageToMXCuBE("Processing started...", "info")
EDUtilsParallel.initializeNbThread()
xsDataResultControlDozor = XSDataResultControlDozor()
# Check if connection to ISPyB needed
if self.dataInput.dataCollectionId is not None:
edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
if self.batchSize is None:
batchSize = ispybDataCollection.numberOfImages
else:
batchSize = self.batchSize
if batchSize > self.maxBatchSize:
batchSize = self.maxBatchSize
if abs(ispybDataCollection.overlap) > 1:
self.hasOverlap = True
self.overlap = ispybDataCollection.overlap
dictImage = self.createImageDictFromISPyB(ispybDataCollection)
else:
# No connection to ISPyB, take parameters from input
if self.dataInput.batchSize is None:
batchSize = self.maxBatchSize
else:
batchSize = self.dataInput.batchSize.value
dictImage = self.createImageDict(self.dataInput)
self.screen("Dozor batch size: {0}".format(batchSize))
if self.dataInput.hdf5BatchSize is not None:
self.hdf5BatchSize = self.dataInput.hdf5BatchSize.value
listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
# Convert HDF5 images to CBF
self.screen("HDF5 converter batch size: {0}".format(self.batchSize))
if self.doRadiationDamage:
self.cbfTempDir = None
else:
self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
listHdf5Batches = self.createListOfBatches(dictImage.keys(), self.batchSize)
dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage, listHdf5Batches, self.doRadiationDamage)
for listBatch in listAllBatches:
# Read the header from the first image in the batch
xsDataFile = dictImage[listBatch[0]]
edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
xsDataInputReadImageHeader = XSDataInputReadImageHeader()
xsDataInputReadImageHeader.image = xsDataFile
edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
edPluginControlReadImageHeader.executeSynchronous()
subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
xsDataInputDozor = XSDataInputDozor()
beam = subWedge.experimentalCondition.beam
detector = subWedge.experimentalCondition.detector
goniostat = subWedge.experimentalCondition.goniostat
xsDataInputDozor.detectorType = detector.type
xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
xsDataInputDozor.spotSize = XSDataInteger(3)
xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
# xsDataInputDozor.fractionPolatization : XSDataDouble optional
orgx = detector.beamPositionY.value / detector.pixelSizeY.value
orgy = detector.beamPositionX.value / detector.pixelSizeX.value
xsDataInputDozor.orgx = XSDataDouble(orgx)
xsDataInputDozor.orgy = XSDataDouble(orgy)
xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
# xsDataInputDozor.imageStep : XSDataDouble optional
xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
xsDataInputDozor.firstImageNumber = subWedge.image[0].number
xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
if self.hasOverlap:
xsDataInputDozor.overlap = XSDataAngle(self.overlap)
strFileName = subWedge.image[0].path.value
strPrefix = EDUtilsImage.getPrefix(strFileName)
strSuffix = EDUtilsImage.getSuffix(strFileName)
if EDUtilsPath.isEMBL():
strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
elif self.hasHdf5Prefix and not self.hasOverlap:
strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
else:
strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
xsDataInputDozor.nameTemplateImage = XSDataString(os.path.join(os.path.dirname(strFileName), strXDSTemplate))
xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
edPluginDozor.dataInput = xsDataInputDozor
edPluginDozor.execute()
edPluginDozor.synchronize()
indexImage = 0
imageDozorBatchList = []
for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
xsDataControlImageDozor = XSDataControlImageDozor()
xsDataControlImageDozor.number = xsDataResultDozor.number
xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
xsDataControlImageDozor.powderWilsonScale = xsDataResultDozor.powderWilsonScale
xsDataControlImageDozor.powderWilsonBfactor = xsDataResultDozor.powderWilsonBfactor
xsDataControlImageDozor.powderWilsonResolution = xsDataResultDozor.powderWilsonResolution
xsDataControlImageDozor.powderWilsonCorrelation = xsDataResultDozor.powderWilsonCorrelation
xsDataControlImageDozor.powderWilsonRfactor = xsDataResultDozor.powderWilsonRfactor
xsDataControlImageDozor.mainScore = xsDataResultDozor.mainScore
xsDataControlImageDozor.spotScore = xsDataResultDozor.spotScore
xsDataControlImageDozor.visibleResolution = xsDataResultDozor.visibleResolution
xsDataControlImageDozor.spotFile = xsDataResultDozor.spotFile
xsDataControlImageDozor.angle = xsDataResultDozor.angle
xsDataResultControlDozor.addImageDozor(xsDataControlImageDozor)
if xsDataResultControlDozor.inputDozor is None:
xsDataResultControlDozor.inputDozor = XSDataDozorInput().parseString(xsDataInputDozor.marshal())
indexImage += 1
dozorSpotListShape = []
dozorSpotList = []
spotFile = None
if xsDataControlImageDozor.spotFile is not None:
spotFile = xsDataControlImageDozor.spotFile.path.value
if os.path.exists(spotFile):
numpyArray = numpy.loadtxt(spotFile, skiprows=3)
dozorSpotList = base64.b64encode(numpyArray.tostring())
dozorSpotListShape.append(numpyArray.shape[0])
if len(numpyArray.shape) > 1:
dozorSpotListShape.append(numpyArray.shape[1])
imageDozorDict = {"index": xsDataControlImageDozor.number.value,
"imageName": xsDataControlImageDozor.image.path.value,
"dozor_score": xsDataControlImageDozor.mainScore.value,
"dozorSpotsNumOf" : xsDataControlImageDozor.spotsNumOf.value,
"dozorSpotFile": spotFile,
"dozorSpotList" : dozorSpotList,
"dozorSpotListShape": dozorSpotListShape,
"dozorSpotsIntAver": xsDataControlImageDozor.spotsIntAver.value,
"dozorSpotsResolution": xsDataControlImageDozor.spotsResolution.value
}
imageDozorBatchList.append(imageDozorDict)
xsDataResultControlDozor.halfDoseTime = edPluginDozor.dataOutput.halfDoseTime
xsDataResultControlDozor.pngPlots = edPluginDozor.dataOutput.pngPlots
self.sendResultToMXCuBE(imageDozorBatchList)
self.sendMessageToMXCuBE("Batch processed")
self.dataOutput = xsDataResultControlDozor
if self.cbfTempDir is not None:
if self.dataInput.keepCbfTmpDirectory is not None and self.dataInput.keepCbfTmpDirectory.value:
self.dataOutput.pathToCbfDirectory = XSDataFile(XSDataString(self.cbfTempDir))
else:
shutil.rmtree(self.cbfTempDir)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_0.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(10000),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
# if self.isFailure():
# return
if len(self.lstInput) == 1:
shutil.copyfile(self.lstInput[0].path.value,
self.dataInput.mergedCurve.path.value)
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for oneFile in self.lstInput[1:]:
edPluginExecAbsoluteFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[0], oneFile])
edPluginExecAbsoluteFidelity.setDataInput(xsd)
edPluginExecAbsoluteFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecAbsoluteFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity
is not None) and (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
edPluginExecRelativeFidelity = self.loadPlugin(
self.__strControlledPluginDatcmp)
xsd = XSDataInputDatcmp(
inputCurve=[self.lstInput[idx + 1], oneFile])
edPluginExecRelativeFidelity.setDataInput(xsd)
edPluginExecRelativeFidelity.connectFAILURE(
self.doFailureExecDatcmp)
edPluginExecRelativeFidelity.connectSUCCESS(
self.doSuccessExecDatcmp)
edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.absoluteFidelity is not None):
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
else:
break
elif (self.relativeFidelity is not None):
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
break
else:
self.lstMerged.append(oneFile)
self.lstSummary.append("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlCharacterisationv1_4.preProcess")
self._xsDataResultCharacterisation = XSDataResultCharacterisation()
# Load the plugins
self._edPluginControlIndexingIndicators = self.loadPlugin(self._strPluginControlIndexingIndicators, \
"Indexing")
self._edPluginExecEvaluationIndexingLABELIT = self.loadPlugin(self._strPluginExecEvaluationIndexing, \
"IndexingEvalualtionLABELIT")
self._edPluginControlIndexingMOSFLM = self.loadPlugin(self._strPluginControlIndexingMOSFLM, \
"IndexingMOSFLM")
self._edPluginExecEvaluationIndexingMOSFLM = self.loadPlugin(self._strPluginExecEvaluationIndexing, \
"IndexingEvalualtionMOSFLM")
self._edPluginControlGeneratePrediction = self.loadPlugin(self._strPluginControlGeneratePrediction, \
"GeneratePrediction")
self._edPluginControlIntegration = self.loadPlugin(self._strPluginControlIntegration, \
"Integration")
self._edPluginControlXDSGenerateBackgroundImage = self.loadPlugin(self._strPluginControlXDSGenerateBackgroundImage, \
"ControlXDSGenerateBackgroundImage")
self._edPluginControlStrategy = self.loadPlugin(self._strPluginControlStrategy, \
"Strategy")
if self._runKappa:
self._edPluginControlKappa = self.loadPlugin(
self._strPluginControlKappaName, "Kappa")
if (self._edPluginControlIndexingIndicators is not None):
self.DEBUG("EDPluginControlCharacterisationv1_4.preProcess: " +
self._strPluginControlIndexingIndicators +
" Found... setting Data Input")
# create Data Input for indexing
xsDataInputCharacterisation = self.getDataInput()
self._xsDataCollection = xsDataInputCharacterisation.getDataCollection(
)
# MXSUP-1445: Check if transmission is less than 10% and warn if it's the case
xsDataFirstSubWedge = self._xsDataCollection.getSubWedge()[0]
xsDataBeam = xsDataFirstSubWedge.getExperimentalCondition(
).getBeam()
if xsDataBeam.getTransmission() is not None:
fTransmission = xsDataBeam.getTransmission().getValue()
if fTransmission < self._fMinTransmission:
strWarningMessageBanner = "^" * 80
strWarningMessage1 = "WARNING! Transmission for characterisation set to %.1f %%" % fTransmission
strWarningMessage2 = "Please consider re-characterising with transmission set to 100 %"
self.warning(strWarningMessageBanner)
self.warning(strWarningMessage1)
self.warning(strWarningMessage2)
self.warning(strWarningMessageBanner)
self.addWarningMessage(strWarningMessageBanner)
self.addWarningMessage(strWarningMessage1)
self.addWarningMessage(strWarningMessage2)
self.addWarningMessage(strWarningMessageBanner)
self.sendMessageToMXCuBE(strWarningMessage1, "warning")
self.sendMessageToMXCuBE(strWarningMessage2, "warning")
xsDataCrystal = None
xsDataSubWedgeList = self._xsDataCollection.getSubWedge()
if ((xsDataSubWedgeList is None) or (xsDataSubWedgeList == [])):
strError = "EDPluginControlCharacterisationv1_4.preProcess: No subwedges in input data."
self.ERROR(strError)
self.setFailure()
else:
# Load the thumbnail plugins
self._iNoReferenceImages = 0
for subWedge in xsDataInputCharacterisation.dataCollection.subWedge:
for image in subWedge.image:
self._iNoReferenceImages += 1
edPluginJpeg = self.loadPlugin(
self._strPluginGenerateThumbnailName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = XSDataFile(image.path)
xsDataInputMXThumbnail.height = XSDataInteger(1024)
xsDataInputMXThumbnail.width = XSDataInteger(1024)
jpegFilename = os.path.splitext(
os.path.basename(image.path.value))[0] + ".jpg"
xsDataInputMXThumbnail.outputPath = XSDataFile(
XSDataString(
os.path.join(self.getWorkingDirectory(),
jpegFilename)))
edPluginJpeg.dataInput = xsDataInputMXThumbnail
edPluginThumnail = self.loadPlugin(
self._strPluginGenerateThumbnailName)
xsDataInputMXThumbnail = XSDataInputMXThumbnail()
xsDataInputMXThumbnail.image = XSDataFile(image.path)
xsDataInputMXThumbnail.height = XSDataInteger(256)
xsDataInputMXThumbnail.width = XSDataInteger(256)
thumbnailFilename = os.path.splitext(
os.path.basename(
image.path.value))[0] + ".thumbnail.jpg"
xsDataInputMXThumbnail.outputPath = XSDataFile(
XSDataString(
os.path.join(self.getWorkingDirectory(),
thumbnailFilename)))
edPluginThumnail.dataInput = xsDataInputMXThumbnail
self._listPluginGenerateThumbnail.append(
(image, edPluginJpeg, edPluginThumnail))
edPluginJpeg.execute()
edPluginThumnail.execute()
xsDataExperimentalCondition = xsDataSubWedgeList[
0].getExperimentalCondition()
# Fix for bug 431: if the flux is zero raise an error
xsDataDoubleFlux = xsDataExperimentalCondition.getBeam(
).getFlux()
if (xsDataDoubleFlux is not None):
if (xsDataDoubleFlux.getValue() < 0.1):
strErrorMessage = "Input flux is negative or close to zero. Execution of characterisation aborted."
self.ERROR(strErrorMessage)
self.sendMessageToMXCuBE(strErrorMessage, "error")
self.addErrorMessage(
"EDPluginControlCharacterisationv1_4.preProcess ERROR: "
+ strErrorMessage)
# self.addComment(strErrorMessage)
self.setFailure()
xsDataDiffractionPlan = self._xsDataCollection.getDiffractionPlan(
)
xsDataStringForcedSpaceGroup = xsDataDiffractionPlan.getForcedSpaceGroup(
)
if (xsDataStringForcedSpaceGroup is not None):
self._xsDataCrystal = XSDataCrystal()
xsDataSpaceGroup = XSDataSpaceGroup()
xsDataSpaceGroup.setName(xsDataStringForcedSpaceGroup)
self._xsDataCrystal.setSpaceGroup(xsDataSpaceGroup)
self._edPluginControlIndexingIndicators.setDataInput(
self._xsDataCollection, "dataCollection")
if self._xsDataCrystal is not None:
self._edPluginControlIndexingIndicators.setDataInput(
self._xsDataCrystal, "crystal")
# Populate characterisation object
self._xsDataResultCharacterisation.setDataCollection(
XSDataCollection.parseString(
self._xsDataCollection.marshal()))
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXia2DIALSv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection(
)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
if EDUtilsPath.isEMBL():
template = ispybDataCollection.fileTemplate.replace(
"%05d", "####")
else:
template = ispybDataCollection.fileTemplate.replace(
"%04d", "####")
if self.dataInput.startFrame is None:
imageNoStart = ispybDataCollection.startImageNumber
else:
imageNoStart = self.dataInput.startFrame.value
if self.dataInput.endFrame is None:
imageNoEnd = imageNoStart + ispybDataCollection.numberOfImages - 1
else:
imageNoEnd = self.dataInput.endFrame.value
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(
directory, ispybDataCollection.fileTemplate % imageNoEnd)
# else:
# directory = self.dataInput.dirN.value
# template = self.dataInput.templateN.value
# imageNoStart = self.dataInput.fromN.value
# imageNoEnd = self.dataInput.toN.value
# fileTemplate = template.replace("####", "%04d")
# pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
# pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(
beamline, "XIA2_DIALS", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(
self.resultsDirectory)
if self.pyarchDirectory is not None:
self.pyarchDirectory = self.pyarchDirectory.replace(
'PROCESSED_DATA', 'RAW_DATA')
if not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "di_{0}_run{1}".format(listPrefix[-3],
listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage
for beamline in ["id23eh1", "id29"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage
for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(
directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(
directory, self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
if EDUtilsPath.isEMBL():
fWaitFileTimeout = 60
else:
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(
XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (
fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(
XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (
fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXia2DIALSAnom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSAnom.anomalous = XSDataBoolean(True)
xsDataInputXia2DIALSAnom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSAnom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSAnom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSAnom.endFrame = XSDataInteger(imageNoEnd)
if self.doNoanom:
xsDataInputXia2DIALSNoanom = XSDataInputXia2DIALS()
xsDataInputXia2DIALSNoanom.anomalous = XSDataBoolean(False)
xsDataInputXia2DIALSNoanom.spaceGroup = self.dataInput.spaceGroup
xsDataInputXia2DIALSNoanom.unitCell = self.dataInput.unitCell
if imageNoStart is not None:
xsDataInputXia2DIALSNoanom.startFrame = XSDataInteger(
imageNoStart)
if imageNoEnd is not None:
xsDataInputXia2DIALSNoanom.endFrame = XSDataInteger(imageNoEnd)
if isH5:
masterFilePath = os.path.join(
directory, self.eiger_template_to_master(template))
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(masterFilePath)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(masterFilePath)))
else:
if self.doAnom:
xsDataInputXia2DIALSAnom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
if self.doNoanom:
xsDataInputXia2DIALSNoanom.addImage(
XSDataFile(XSDataString(pathToStartImage)))
self.timeStart = time.localtime()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to running
if self.doAnom:
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingCommandLine=self.processingCommandLine,
processingPrograms=self.processingPrograms,
isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXia2DIALSAnom.dataInput = xsDataInputXia2DIALSAnom
self.edPluginExecXia2DIALSAnom.execute()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.dataInput = xsDataInputXia2DIALSNoanom
self.edPluginExecXia2DIALSNoanom.execute()
if self.doAnom:
self.edPluginExecXia2DIALSAnom.synchronize()
if self.doNoanom:
self.edPluginExecXia2DIALSNoanom.synchronize()
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.doAnom:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSAnom, True, proposal,
self.autoProcProgramIdAnom, self.autoProcIntegrationIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(
self.edPluginExecXia2DIALSNoanom, False, proposal,
self.autoProcProgramIdNoanom, self.autoProcIntegrationIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def testSetDataModelInput(self):
# Crystal
from XSDataRaddosev10 import XSDataRaddoseInput
xsDataRaddoseInput = XSDataRaddoseInput()
from XSDataCommon import XSDataString
from XSDataCommon import XSDataDouble
from XSDataCommon import XSDataAngle
from XSDataCommon import XSDataLength
from XSDataCommon import XSDataSize
from XSDataCommon import XSDataInteger
from XSDataRaddosev10 import XSDataCell
from XSDataRaddosev10 import XSDataAtom
from XSDataRaddosev10 import XSDataAtomicComposition
xsDataAtomSulfur = XSDataAtom()
xsDataAtomSulfur.setNumberOf(XSDataDouble(4))
xsDataAtomSulfur.setSymbol(XSDataString("S"))
xsDataAtomSelenium = XSDataAtom()
xsDataAtomSelenium.setNumberOf(XSDataDouble(4))
xsDataAtomSelenium.setSymbol(XSDataString("Se"))
xsDataAtomicComposition = XSDataAtomicComposition()
xsDataAtomicComposition.addAtom(xsDataAtomSulfur)
xsDataAtomicComposition.addAtom(xsDataAtomSelenium)
xsDataRaddoseInput.setCrystalPATM(xsDataAtomicComposition)
xsDataRaddoseInput.setCrystalNRES(XSDataInteger(295))
xsDataRaddoseInput.setCrystalNMON(XSDataInteger(8))
xsDataCell = XSDataCell(angle_alpha=XSDataAngle(90.0),
angle_beta=XSDataAngle(90.0),
angle_gamma=XSDataAngle(90.0),
length_a=XSDataLength(78.9),
length_b=XSDataLength(95.162),
length_c=XSDataLength(104.087))
xsDataSizeCrystal = XSDataSize(XSDataLength(0.1), XSDataLength(0.1),
XSDataLength(0.1))
xsDataRaddoseInput.setCrystalCell(xsDataCell)
xsDataRaddoseInput.setCrystalSize(xsDataSizeCrystal)
# Beam
from XSDataCommon import XSDataFlux
from XSDataCommon import XSDataWavelength
from XSDataCommon import XSDataTime
xsDataSize = XSDataSize(x=XSDataLength(0.1), y=XSDataLength(0.1))
xsDataRaddoseInput.setBeamSize(xsDataSize)
xsDataRaddoseInput.setBeamFlux(XSDataFlux(1e+12))
xsDataRaddoseInput.setBeamWavelength(XSDataWavelength(2.41))
xsDataRaddoseInput.setBeamExposureTime(XSDataTime(1))
xsDataRaddoseInput.setNumberOfImages(XSDataInteger(1))
xsDataRaddoseInput.exportToFile(self.strObtainedInputFile)
strExpectedInput = self.readAndParseFile(self.strReferenceInputFile)
strObtainedInput = self.readAndParseFile(self.strObtainedInputFile)
xsDataRaddoseInputExpected = XSDataRaddoseInput.parseString(
strExpectedInput)
xsDataRaddoseInputObtained = XSDataRaddoseInput.parseString(
strObtainedInput)
EDAssert.equal(xsDataRaddoseInputExpected.marshal(),
xsDataRaddoseInputObtained.marshal())
def testIsSameExperimentalCondition(self):
edPluginSubWedgeMergev10 = self.createPlugin()
xsDataExperimentalConditionReference = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionSameAsReference = self.getTestExperimentalCondition(
)
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionSameAsReference), True)
xsDataExperimentalConditionDifferentExposureTime = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentExposureTime.getBeam(
).setExposureTime(XSDataTime(10.0))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentExposureTime), False)
xsDataExperimentalConditionDifferentWavelength = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentWavelength.getBeam().setWavelength(
XSDataWavelength(1.5))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentWavelength), False)
xsDataExperimentalConditionDifferentBeamPositionX = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentBeamPositionX.getDetector(
).setBeamPositionX(XSDataLength(20.0))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentBeamPositionX), False)
xsDataExperimentalConditionDifferentBeamPositionY = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentBeamPositionY.getDetector(
).setBeamPositionY(XSDataLength(20.0))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentBeamPositionY), False)
xsDataExperimentalConditionDifferentDistance = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentDistance.getDetector().setDistance(
XSDataLength(220.0))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentDistance), False)
xsDataExperimentalConditionDifferentName = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentName.getDetector().setName(
XSDataString(u"EDNA"))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentName), False)
xsDataExperimentalConditionDifferentNumberPixelX = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentNumberPixelX.getDetector(
).setNumberPixelX(XSDataInteger(2))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentNumberPixelX), False)
xsDataExperimentalConditionDifferentNumberPixelY = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentNumberPixelY.getDetector(
).setNumberPixelY(XSDataInteger(2))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentNumberPixelY), False)
xsDataExperimentalConditionDifferentSerialNumber = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentSerialNumber.getDetector(
).setSerialNumber(XSDataString(u"EDNA"))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentSerialNumber), False)
xsDataExperimentalConditionDifferentTwoTheta = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentTwoTheta.getDetector().setTwoTheta(
XSDataAngle(90.0))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentTwoTheta), False)
xsDataExperimentalConditionDifferentOscillationWidth = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentOscillationWidth.getGoniostat(
).setOscillationWidth(XSDataAngle(2.0))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentOscillationWidth), False)
xsDataExperimentalConditionDifferentRotationAxis = self.getTestExperimentalCondition(
)
xsDataExperimentalConditionDifferentRotationAxis.getGoniostat(
).setRotationAxis(XSDataString(u"EDNA"))
EDAssert.equal(
edPluginSubWedgeMergev10.isSameExperimentalCondition(
xsDataExperimentalConditionReference,
xsDataExperimentalConditionDifferentRotationAxis), False)
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginBioSaxsSmartMergev1_7.process")
xsdwf = XSDataInputWaitMultiFile(
timeOut=XSDataTime(30),
expectedSize=XSDataInteger(self.minimumCurveFileSize),
expectedFile=[XSDataFile(i.path) for i in self.lstInput])
self.__edPluginExecWaitFile.setDataInput(xsdwf)
self.__edPluginExecWaitFile.connectFAILURE(self.doFailureExecWait)
self.__edPluginExecWaitFile.connectSUCCESS(self.doSuccessExecWait)
self.__edPluginExecWaitFile.executeSynchronous()
if self.isFailure():
return
if len(self.lstInput) == 1:
inp = self.lstInput[0].path.value
dst = self.dataInput.mergedCurve.path.value
if not os.path.isdir(os.path.dirname(dst)):
self.error("Output directory for %s does not exist" % dst)
os.makedirs(os.path.dirname(dst))
if not os.path.exists(inp):
self.warning("Input %s does not (yet?) exist" % inp)
time.sleep(1.0)
shutil.copyfile(inp, dst)
self.addExecutiveSummaryLine(
"Got only one frame ... nothing to merge !!!")
else:
self.lstMerged = []
if (self.absoluteFidelity is not None) or (self.relativeFidelity
is not None):
if self.absoluteFidelity is not None:
for idx, oneFile in enumerate(self.lstInput[1:]):
self.DEBUG("Calculating similarity of 0 and %s" % idx)
self.datcmp(self.lstInput[0], oneFile)
# edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], oneFile])
# edPluginExecAbsoluteFidelity.setDataInput(xsd)
# edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecAbsoluteFidelity.execute()
if (self.relativeFidelity is not None):
if (self.absoluteFidelity is None):
self.DEBUG("Calculating similarity of 0 and 1")
self.datcmp(self.lstInput[0], self.lstInput[1])
# edPluginExecAbsoluteFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[0], self.lstInput[1] ])
# edPluginExecAbsoluteFidelity.setDataInput(xsd)
# edPluginExecAbsoluteFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecAbsoluteFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecAbsoluteFidelity.execute()
if (len(self.lstInput) > 2):
for idx, oneFile in enumerate(self.lstInput[2:]):
self.DEBUG("Calculating similarity of %s and %s" %
(idx, idx + 1))
self.datcmp(self.lstInput[idx + 1], oneFile)
# edPluginExecRelativeFidelity = self.loadPlugin(self.__strControlledPluginDatcmp)
# xsd = XSDataInputDatcmp(inputCurve=[self.lstInput[idx + 1], oneFile])
# edPluginExecRelativeFidelity.setDataInput(xsd)
# edPluginExecRelativeFidelity.connectFAILURE(self.doFailureExecDatcmp)
# edPluginExecRelativeFidelity.connectSUCCESS(self.doSuccessExecDatcmp)
# edPluginExecRelativeFidelity.execute()
self.synchronizePlugins()
for idx, oneFile in enumerate(self.lstInput):
if idx == 0:
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity
is not None) and (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity) and (self.dictSimilarities[
(idx - 1, idx)] >= self.relativeFidelity):
self.lstMerged.append(oneFile)
elif (self.absoluteFidelity is not None):
if (0, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (0, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(0, idx)] >=
self.absoluteFidelity):
self.lstMerged.append(oneFile)
elif (self.relativeFidelity is not None):
if (idx - 1, idx) not in self.dictSimilarities:
self.ERROR("dict missing %i,%i: \n" % (idx - 1, idx) +
"\n".join([
"%s: %s" %
(key, self.dictSimilarities[key])
for key in self.dictSimilarities
]))
self.resynchronize()
if (self.dictSimilarities[(idx - 1, idx)] >=
self.relativeFidelity):
self.lstMerged.append(oneFile)
else:
self.lstMerged.append(oneFile)
self.lstMerged.sort(cmp)
if len(self.lstMerged) != len(self.lstInput):
self.strRadiationDamage = "Radiation damage detected, merged %i curves" % len(
self.lstMerged)
self.WARNING(self.strRadiationDamage)
self.addExecutiveSummaryLine("WARNING: " +
self.strRadiationDamage)
self.addExecutiveSummaryLine("Merging files: " + " ".join(
[os.path.basename(i.path.value) for i in self.lstMerged]))
if len(self.lstMerged) == 1:
self.rewriteHeader(self.lstMerged[0].path.value,
self.strMergedFile)
else:
self.__edPluginExecDataver = self.loadPlugin(
self.__strControlledPluginDataver)
xsd = XSDataInputDataver(inputCurve=self.lstMerged)
# outputCurve=self.dataInput.mergedCurve,
self.__edPluginExecDataver.setDataInput(xsd)
self.__edPluginExecDataver.connectSUCCESS(
self.doSuccessExecDataver)
self.__edPluginExecDataver.connectFAILURE(
self.doFailureExecDataver)
self.__edPluginExecDataver.executeSynchronous()
if (self.fConcentration == 0) and (self.strSubFile is not None):
if (self.__class__.lastBuffer
is not None) and (self.__class__.lastSample is not None):
self.__edPluginExecAutoSub = self.loadPlugin(
self.__strControlledPluginAutoSub)
base = "_".join(
os.path.basename(
self.__class__.lastSample.path.value).split("_")[:-1])
suff = os.path.basename(self.strSubFile).split("_")[-1]
sub = os.path.join(os.path.dirname(self.strSubFile),
base + "_" + suff)
self.xsdSubtractedCurve = XSDataFile(XSDataString(sub))
#self.curves.append(xsdSubtractedCurve)
self.__edPluginExecAutoSub.dataInput = XSDataInputAutoSub(
sampleCurve=self.__class__.lastSample,
buffers=[
self.__class__.lastBuffer, self.dataInput.mergedCurve
],
subtractedCurve=self.xsdSubtractedCurve)
self.__edPluginExecAutoSub.connectSUCCESS(
self.doSuccessExecAutoSub)
self.__edPluginExecAutoSub.connectFAILURE(
self.doFailureExecAutoSub)
self.__edPluginExecAutoSub.executeSynchronous()
if self.isFailure():
return
self.__edPluginSaxsAnalysis = self.loadPlugin(
self.__strControlledPluginSaxsAnalysis)
self.__edPluginSaxsAnalysis.dataInput = XSDataInputSaxsAnalysis(
scatterCurve=self.xsdSubtractedCurve,
autoRg=self.autoRg,
graphFormat=XSDataString("png"))
self.__edPluginSaxsAnalysis.connectSUCCESS(
self.doSuccessSaxsAnalysis)
self.__edPluginSaxsAnalysis.connectFAILURE(
self.doFailureSaxsAnalysis)
self.__edPluginSaxsAnalysis.executeSynchronous()
self.__class__.lastBuffer = self.dataInput.mergedCurve
#self.__class__.lastSample = None #Information neededfor transfer to ISPyB
self.forgetLastSample = True
else:
self.__class__.lastSample = self.dataInput.mergedCurve
if self.dataInput.sample and self.dataInput.sample.login and self.dataInput.sample.passwd and self.dataInput.sample.measurementID:
self.addExecutiveSummaryLine("Registering to ISPyB")
self.lstDiscarded = list(set(self.lstInput) - set(self.lstMerged))
self.__class__.dictFrames[self.dataInput.mergedCurve] = {
'averaged': self.lstMerged,
'discarded': self.lstDiscarded
}
self.__edPluginSaxsISPyB = self.loadPlugin(
self.__strControlledPluginSaxsISPyB)
if len(self.lstInput) > 1:
frameAverage = XSDataInteger(len(self.lstInput))
frameMerged = XSDataInteger(len(self.lstMerged))
else:
frameMerged = frameAverage = XSDataInteger(1)
self.curves = [XSDataFile(i.path) for i in self.lstInput]
self.discardedCurves = [
XSDataFile(i.path) for i in self.lstDiscarded
]
self.mergedCurves = [XSDataFile(i.path) for i in self.lstMerged]
averageFilePath = None
if self.strMergedFile is not None:
averageFilePath = XSDataFile(XSDataString(self.strMergedFile))
self.sampleFrames = self.getAveragedFrameByFilename(
self.__class__.lastSample)
lastSample = None
if self.__class__.lastSample is not None:
lastSample = self.__class__.lastSample
subtractedCurve = None
if self.xsdSubtractedCurve is not None:
subtractedCurve = self.xsdSubtractedCurve
else:
subtractedCurve = None
xsdin = XSDataInputBioSaxsISPyBv1_0(
sample=self.dataInput.sample,
autoRg=self.autoRg,
gnom=self.gnom,
volume=self.volume,
frameAverage=frameAverage,
frameMerged=frameMerged,
curves=self.curves,
discardedFrames=self.discardedCurves,
averagedFrames=self.mergedCurves,
averageFilePath=averageFilePath,
bufferFrames=self.bufferFrames,
sampleFrames=self.sampleFrames,
bestBuffer=self.xsBestBuffer,
averageSample=lastSample,
scatterPlot=self.xsScatterPlot,
guinierPlot=self.xsGuinierPlot,
kratkyPlot=self.xsKratkyPlot,
densityPlot=self.xsDensityPlot,
subtractedFilePath=subtractedCurve
# destination=self.dataInput.sample.ispybDestination #duplicate, already in sample
)
self.__edPluginSaxsISPyB.dataInput = xsdin
self.__edPluginSaxsISPyB.connectSUCCESS(self.doSuccessISPyB)
self.__edPluginSaxsISPyB.connectFAILURE(self.doFailureISPyB)
self.__edPluginSaxsISPyB.execute()
# transfer analysis data to correct location on nice
if self.gnom is not None:
self.outdir = os.path.join(
os.path.dirname(os.path.dirname(self.lstStrInput[0])),
"ednaAnalysis")
basename = os.path.basename(
os.path.splitext(self.gnom.gnomFile.path.value)[0])
self.outdir = os.path.join(self.outdir, basename)
if not os.path.isdir(self.outdir):
os.makedirs(self.outdir)
#self.outFile = os.path.join(outdir, "NoResults.html")
workingdir = os.path.dirname(self.gnom.gnomFile.path.value)
self.pluginRsync = self.loadPlugin(self.cpRsync)
self.pluginRsync.dataInput = XSDataInputRsync(
source=XSDataFile(XSDataString(workingdir)),
destination=XSDataFile(XSDataString(self.outdir)),
options=XSDataString("-avx"))
self.pluginRsync.connectSUCCESS(self.doSuccessExecRsync)
self.pluginRsync.connectFAILURE(self.doFailureExecRsync)
self.pluginRsync.executeSynchronous()
if self.forgetLastSample:
#Also redefine dictionary to contain the buffer just processed?
self.__class__.lastSample = None
def getXSDataRaddoseInput(self, _xsDataBeam, _xsDataSample, _inumOperators,
_iNumberOfImages):
xsDataRaddoseInput = XSDataRaddoseInput()
# Beam
xsDataRaddoseInput.setBeamSize(_xsDataBeam.getSize())
xsDataRaddoseInput.setBeamFlux(_xsDataBeam.getFlux())
xsDataRaddoseInput.setBeamWavelength(_xsDataBeam.getWavelength())
xsDataRaddoseInput.setBeamExposureTime(_xsDataBeam.getExposureTime())
xsDataRaddoseInput.setNumberOfImages(XSDataInteger(_iNumberOfImages))
xsDataRaddoseInput.setCrystalCell(_xsDataSample.getCrystal().getCell())
xsDataRaddoseInput.setCrystalSize(_xsDataSample.getSize())
xsDataComposition = _xsDataSample.getChemicalComposition()
xsDataSolvent = xsDataComposition.getSolvent()
if (xsDataSolvent is not None):
xsDataRaddoseInput.setCrystalSATM(xsDataSolvent.getAtoms())
xsDataStructure = xsDataComposition.getStructure()
if (xsDataStructure is not None):
xsDataChains = xsDataStructure.getChain()
totalNRESInStructure = 0
totalNDNAInStructure = 0
totalNRNAInStructure = 0
totalPATM = XSDataAtomicComposition()
for chain in xsDataChains:
# heavy atoms of each chain to be added in the PATM
xsDataAtomicCompositionHeavyAtoms = chain.getHeavyAtoms()
if (xsDataAtomicCompositionHeavyAtoms is not None):
iterator = 1
while iterator <= chain.getNumberOfCopies().getValue():
totalPATM = self.mergeAtomicComposition(
totalPATM, xsDataAtomicCompositionHeavyAtoms)
iterator = iterator + 1
type = chain.getType().getValue()
numberOfMonomers = chain.getNumberOfMonomers().getValue(
) * chain.getNumberOfCopies().getValue()
if (type == "protein"):
totalNRESInStructure = totalNRESInStructure + numberOfMonomers
elif (type == "dna"):
totalNDNAInStructure = totalNDNAInStructure + numberOfMonomers
elif (type == "rna"):
totalNRNAInStructure = totalNRNAInStructure + numberOfMonomers
xsDataLigands = xsDataStructure.getLigand()
for ligand in xsDataLigands:
# Light atoms to be added to the NRES
nres = ligand.getNumberOfLightAtoms().getValue(
) * ligand.getNumberOfCopies().getValue() / 7.85
totalNRESInStructure = totalNRESInStructure + nres
# Heavy atoms to be added to the PATM
if (ligand.getHeavyAtoms() is not None):
iterator = 1
while iterator <= ligand.getNumberOfCopies().getValue():
totalPATM = self.mergeAtomicComposition(
totalPATM, ligand.getHeavyAtoms())
iterator = iterator + 1
if (totalNRESInStructure != 0):
xsDataRaddoseInput.setCrystalNRES(
XSDataInteger(int(round(totalNRESInStructure))))
if (totalNDNAInStructure != 0):
xsDataRaddoseInput.setCrystalNDNA(
XSDataInteger(int(totalNDNAInStructure)))
if (totalNRNAInStructure != 0):
xsDataRaddoseInput.setCrystalNRNA(
XSDataInteger(int(totalNRNAInStructure)))
if (len(totalPATM.getAtom()) != 0):
xsDataRaddoseInput.setCrystalPATM(totalPATM)
xsDataNumberNumStructInAU = xsDataStructure.getNumberOfCopiesInAsymmetricUnit(
)
xsDataNumberNumStructInUC = int(
xsDataNumberNumStructInAU.getValue() * _inumOperators)
xsDataRaddoseInput.setCrystalNMON(
XSDataInteger(xsDataNumberNumStructInUC))
return xsDataRaddoseInput
def create_processing_input(self, data_collection):
"""Creates dozor input file base on data collection parameters
:param data_collection: data collection object
:type : queue_model_objects.DataCollection
"""
acquisition = data_collection.acquisitions[0]
acq_params = acquisition.acquisition_parameters
input_file = XSDataInputControlDozor()
image_file_template = "%s_%%d_%%05d.cbf" % (
acquisition.path_template.get_prefix())
template = os.path.join(acquisition.path_template.directory,
image_file_template)
first_image_num = acq_params.first_image
images_num = acq_params.num_images
last_image_num = first_image_num + images_num - 1
run_number = acquisition.path_template.run_number
lines_num = acq_params.num_lines
pixel_min = 0
pixel_max = 0
beamstop_size = 0
beamstop_distance = 0
beamstop_direction = 0
pixel_min = self.detector_hwobj.get_pixel_min()
pixel_max = self.detector_hwobj.get_pixel_max()
#?utilite du beamstop ???????????????
beamstop_size = self.beamstop_hwobj.get_size()
beamstop_distance = self.beamstop_hwobj.get_distance()
beamstop_direction = self.beamstop_hwobj.get_direction()
if data_collection.grid:
grid_params = data_collection.grid.get_properties()
reversing_rotation = grid_params["reversing_rotation"]
else:
reversing_rotation = False
self.params_dict["template"] = template
self.params_dict["first_image_num"] = first_image_num
self.params_dict["images_num"] = images_num
self.params_dict["lines_num"] = lines_num
self.params_dict["images_per_line"] = images_num / lines_num
self.params_dict["run_number"] = run_number
self.params_dict["pixel_min"] = pixel_min
self.params_dict["pixel_max"] = pixel_max
self.params_dict["beamstop_size"] = beamstop_size
self.params_dict["beamstop_distance"] = beamstop_distance
self.params_dict["beamstop_direction"] = beamstop_direction
self.params_dict["status"] = "Started"
self.params_dict["title"] = "%s_%d_#####.cbf (%d - %d)" % \
(acquisition.path_template.get_prefix(),
acquisition.path_template.run_number,
first_image_num,
last_image_num)
self.params_dict["comments"] = "Scan lines: %d, frames per line: %d" % \
(lines_num, images_num / lines_num)
if lines_num > 1:
self.params_dict["dx_mm"] = grid_params["dx_mm"]
self.params_dict["dy_mm"] = grid_params["dy_mm"]
self.params_dict["steps_x"] = grid_params["steps_x"]
self.params_dict["steps_y"] = grid_params["steps_y"]
self.params_dict["xOffset"] = grid_params["xOffset"]
self.params_dict["yOffset"] = grid_params["yOffset"]
else:
self.params_dict["steps_y"] = 1
input_file.setTemplate(XSDataString(template))
input_file.setFirst_image_number(XSDataInteger(first_image_num))
input_file.setLast_image_number(XSDataInteger(last_image_num))
input_file.setFirst_run_number(XSDataInteger(run_number))
input_file.setLast_run_number(XSDataInteger(run_number))
input_file.setLine_number_of(XSDataInteger(lines_num))
input_file.setReversing_rotation(XSDataBoolean(reversing_rotation))
input_file.setPixelMin(XSDataInteger(pixel_min))
input_file.setPixelMax(XSDataInteger(pixel_max))
input_file.setBeamstopSize(XSDataDouble(beamstop_size))
input_file.setBeamstopDistance(XSDataDouble(beamstop_distance))
input_file.setBeamstopDirection(XSDataString(beamstop_direction))
return input_file
def populateXSDataInputSPDCake(self, _inputDict=None):
self.DEBUG("EDPluginControlID11v1_0.populateXSDataInputSPDCake")
xsDataInputSPDCake = XSDataInputSPDCake()
if isinstance(_inputDict, dict):
self.__dictID11 = _inputDict
# Angle of tilt
if "ANGLE OF TILT" in self.__dictID11:
xsDataTilt = XSDataAngle()
xsDataTilt.setValue(float(self.__dictID11["ANGLE OF TILT"]))
xsDataTilt.setUnit(XSDataString("deg"))
xsDataInputSPDCake.setAngleOfTilt(xsDataTilt)
if "TILT ROTATION" in self.__dictID11:
xsDataTiltRot = XSDataAngle()
xsDataTiltRot.setValue(float(self.__dictID11["TILT ROTATION"]))
xsDataTiltRot.setUnit(XSDataString("deg"))
xsDataInputSPDCake.setTiltRotation(xsDataTiltRot)
if "X-PIXEL SIZE" in self.__dictID11:
xsDataXPixel = XSDataLength()
xsDataXPixel.setValue(float(self.__dictID11["Y-PIXEL SIZE"]))
xsDataXPixel.setUnit(XSDataString("micron"))
xsDataInputSPDCake.setPixelSizeX(xsDataXPixel)
if "Y-PIXEL SIZE" in self.__dictID11:
xsDataYPixel = XSDataLength()
xsDataYPixel.setValue(float(self.__dictID11["Y-PIXEL SIZE"]))
xsDataYPixel.setUnit(XSDataString("micron"))
xsDataInputSPDCake.setPixelSizeY(xsDataYPixel)
if "DISTANCE" in self.__dictID11:
xsDataDistance = XSDataLength()
xsDataDistance.setValue(float(self.__dictID11["DISTANCE"]))
xsDataDistance.setUnit(XSDataString("mm"))
xsDataInputSPDCake.setSampleToDetectorDistance(xsDataDistance)
if "WAVELENGTH" in self.__dictID11:
xsDataWaweLength = XSDataWavelength()
xsDataWaweLength.setValue(float(self.__dictID11["WAVELENGTH"]))
xsDataWaweLength.setUnit(XSDataString("A"))
xsDataInputSPDCake.setWavelength(xsDataWaweLength)
if "DIM1_DATA" in self.__dictID11:
xsDataBufferSizeX = XSDataInteger(int(
self.__dictID11["DIM1_DATA"]))
xsDataInputSPDCake.setBufferSizeX(xsDataBufferSizeX)
if "DIM2_DATA" in self.__dictID11:
xsDataBufferSizeY = XSDataInteger(int(
self.__dictID11["DIM2_DATA"]))
xsDataInputSPDCake.setBufferSizeY(xsDataBufferSizeY)
# Dark current
if ("DARK CURRENT"
in self.__dictID11) and (self.__dictID11["DARK CURRENT"]
== "YES"):
if ("DC FILE" in self.__dictID11) and os.path.isfile(
self.__dictID11["DC FILE"]):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__dictID11["DC FILE"]))
xsDataInputSPDCake.setDarkCurrentImageFile(xsDataFile)
else:
self.warning(
"Asked for DC Current correction but no DC current file")
if ("FLAT-FIELD" in self.__dictID11) and (self.__dictID11["FLAT-FIELD"]
== "YES"):
if ("FF FILE" in self.__dictID11) and os.path.isfile(
self.__dictID11["FF FILE"]):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__dictID11["FF FILE"]))
xsDataInputSPDCake.setFlatFieldImageFile(xsDataFile)
else:
self.warning(
"Asked for FLAT-FIELD correction but no FLAT-FIELD file")
if ("MASK FILE" in self.__dictID11) and (self.__dictID11["USE MASK"]
== "YES"):
if ("MASK FILE" in self.__dictID11) and os.path.isfile(
self.__dictID11["MASK FILE"]):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__dictID11["MASK FILE"]))
xsDataInputSPDCake.setMaskFile(xsDataFile)
else:
self.warning(
"Asked for DC Current correction but no DC current file")
if ("FF SCALE" in self.__dictID11) and (self.__dictID11["FF SCALE"]
== "YES"):
if ("FF MULTIPLIER" in self.__dictID11):
try:
value = float(self.__dictID11["FF MULTIPLIER"])
except Exception:
self.warning(
"Asked for FF SCALE correction but FF MULTIPLIER provided (%s) in not float !"
% (self.__dictID11["FF MULTIPLIER"]))
else:
xsDataInputSPDCake.setIntensityScaleFactor(
XSDataDouble(1 / value))
else:
self.warning(
"Asked for FF SCALE correction but no FF MULTIPLIER provided"
)
if ("SPATIAL DIS."
in self.__dictID11) and (self.__dictID11["SPATIAL DIS."]
== "YES"):
if ("SD FILE" in self.__dictID11) and os.path.isfile(
self.__dictID11["SD FILE"]):
xsDataFile = XSDataFile()
xsDataFile.setPath(XSDataString(self.__dictID11["SD FILE"]))
xsDataInputSPDCake.setSpatialDistortionFile(xsDataFile)
else:
self.warning(
"Asked for SPATIAL DISTORSION correction but no SPATIAL DISTORSION file"
)
if "START AZIMUTH" in self.__dictID11:
xsDataAzimuthStart = XSDataAngle()
xsDataAzimuthStart.setValue(float(
self.__dictID11["START AZIMUTH"]))
xsDataAzimuthStart.setUnit(XSDataString("deg"))
xsDataInputSPDCake.setStartAzimuth(xsDataAzimuthStart)
if "END AZIMUTH" in self.__dictID11:
xsDataAzimuthStop = XSDataAngle()
xsDataAzimuthStop.setValue(float(self.__dictID11["END AZIMUTH"]))
xsDataAzimuthStop.setUnit(XSDataString("deg"))
xsDataInputSPDCake.setStopAzimuth(xsDataAzimuthStop)
if "AZIMUTH BINS" in self.__dictID11:
xsDataAzimuthStep = XSDataAngle()
xsDataAzimuthStep.setValue(
(float(self.__dictID11["END AZIMUTH"]) -
float(self.__dictID11["START AZIMUTH"])) /
float(self.__dictID11["AZIMUTH BINS"]))
xsDataAzimuthStep.setUnit(XSDataString("deg"))
xsDataInputSPDCake.setStepAzimuth(xsDataAzimuthStep)
if "INNER RADIUS" in self.__dictID11:
xsDataInnerRadius = XSDataDouble()
xsDataInnerRadius.setValue(float(self.__dictID11["INNER RADIUS"]))
xsDataInputSPDCake.setInnerRadius(xsDataInnerRadius)
if "OUTER RADIUS" in self.__dictID11:
xsDataOuterRadius = XSDataDouble()
xsDataOuterRadius.setValue(float(self.__dictID11["OUTER RADIUS"]))
xsDataInputSPDCake.setOuterRadius(xsDataOuterRadius)
if "X-BEAM CENTRE" in self.__dictID11:
xsDataXBeamCentre = XSDataDouble()
xsDataXBeamCentre.setValue(float(self.__dictID11["X-BEAM CENTRE"]))
xsDataInputSPDCake.setBeamCentreInPixelsX(xsDataXBeamCentre)
if "Y-BEAM CENTRE" in self.__dictID11:
xsDataYBeamCentre = XSDataDouble()
xsDataYBeamCentre.setValue(float(self.__dictID11["Y-BEAM CENTRE"]))
xsDataInputSPDCake.setBeamCentreInPixelsY(xsDataYBeamCentre)
# if "saving_format" in self.__dictID11:
# xsSaveFormat = XSDataString()
# if self.__dictID11["saving_format"] == "SPREAD SHEET":
# xsSaveFormat.setValue("spr")
# elif self.__dictID11["saving_format"] == "CIF":
# xsSaveFormat.setValue("cif")
# elif self.__dictID11["saving_format"] == "CHIPLOT":
# xsSaveFormat.setValue("chi")
# else:
# xsSaveFormat.setValue("edf")
# xsDataInputSPDCake.setOutputFileType(xsSaveFormat)
if "output_dir" in self.__dictID11:
xsOutputDir = XSDataFile()
xsOutputDir.setPath(XSDataString(self.__dictID11["output_dir"]))
xsDataInputSPDCake.setOutputDir(xsOutputDir)
#Default options to SPD
xsDataInputSPDCake.setOutputFileType(XSDataString("azim"))
xsDataInputSPDCake.setDeleteCorImg(
XSDataBoolean(not self.isVerboseDebug()))
xsDataInputSPDCake.setCorrectTiltMask(XSDataBoolean(self.bCorrectMask))
return xsDataInputSPDCake
def buildChildren(self, child_, nodeName_):
if child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'experimentalCondition':
obj_ = XSDataExperimentalCondition()
obj_.build(child_)
self.setExperimentalCondition(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'diffractionPlan':
obj_ = XSDataDiffractionPlan()
obj_.build(child_)
self.setDiffractionPlan(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'sample':
obj_ = XSDataSampleCrystalMM()
obj_.build(child_)
self.setSample(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'imagePath':
obj_ = XSDataFile()
obj_.build(child_)
self.imagePath.append(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'flux':
obj_ = XSDataFloat()
obj_.build(child_)
self.setFlux(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'minExposureTimePerImage':
obj_ = XSDataTime()
obj_.build(child_)
self.setMinExposureTimePerImage(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSize':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSize(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSizeX':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSizeX(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamSizeY':
obj_ = XSDataLength()
obj_.build(child_)
self.setBeamSizeY(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'apertureSize':
obj_ = XSDataLength()
obj_.build(child_)
self.setApertureSize(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'templateMode':
obj_ = XSDataBoolean()
obj_.build(child_)
self.setTemplateMode(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'generatedTemplateFile':
obj_ = XSDataFile()
obj_.build(child_)
self.setGeneratedTemplateFile(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'resultsFilePath':
obj_ = XSDataFile()
obj_.build(child_)
self.setResultsFilePath(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamPosX':
obj_ = XSDataFloat()
obj_.build(child_)
self.setBeamPosX(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'beamPosY':
obj_ = XSDataFloat()
obj_.build(child_)
self.setBeamPosY(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'wavelength':
obj_ = XSDataWavelength()
obj_.build(child_)
self.setWavelength(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'transmission':
obj_ = XSDataDouble()
obj_.build(child_)
self.setTransmission(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'dataCollectionId':
obj_ = XSDataInteger()
obj_.build(child_)
self.setDataCollectionId(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'shortComments':
obj_ = XSDataString()
obj_.build(child_)
self.setShortComments(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'comments':
obj_ = XSDataString()
obj_.build(child_)
self.setComments(obj_)
elif child_.nodeType == Node.ELEMENT_NODE and \
nodeName_ == 'inputCharacterisation':
obj_ = XSDataInputCharacterisation()
obj_.build(child_)
self.setInputCharacterisation(obj_)
def create_input_files(self, xds_dir, mosflm_dir, dc_pars):
fileinfo = dc_pars["fileinfo"]
osc_seq = dc_pars["oscillation_sequence"][0]
prefix = fileinfo["prefix"]
runno = fileinfo["run_number"]
exp_time = osc_seq["exposure_time"]
start_angle = osc_seq["start"]
nb_images = osc_seq["number_of_images"]
start_img_num = osc_seq["start_image_number"]
angle_increment = osc_seq["range"]
wavelength = osc_seq.get("wavelength", 0)
xds_template_name = "XDS_TEMPLATE.INP"
mosflm_template_name = "mosflm_template.dat"
xds_template_path = os.path.join(self.template_dir, xds_template_name)
mosflm_template_path = os.path.join(self.template_dir, mosflm_template_name)
xds_file = os.path.join(xds_dir, "XDS.INP")
mosflm_file = os.path.join(mosflm_dir, "mosflm.dat")
t = datetime.now()
# PREPARE VARIABLES
detsamdis = self.var_ds.detsamdis
beamx, beamy = self.var_ds.beamx, self.var_ds.beamy
mbeamx, mbeamy = beamy * 0.172, beamx * 0.172
datarangestartnum = start_img_num
datarangefinishnum = start_img_num + nb_images - 1
backgroundrangestartnum = start_img_num
spotrangestartnum = start_img_num
if angle_increment != 0:
minimumrange = int(round(20 / angle_increment))
elif angle_increment == 0:
minimumrange = 1
if nb_images >= minimumrange:
backgroundrangefinishnum = start_img_num + minimumrange - 1
if nb_images >= minimumrange:
spotrangefinishnum = start_img_num + minimumrange - 1
if nb_images < minimumrange:
backgroundrangefinishnum = start_img_num + nb_images - 1
if nb_images < minimumrange:
spotrangefinishnum = start_img_num + nb_images - 1
testlowres = 8.0
largestvector = (
0.172
* ((max(beamx, 2463 - beamx)) ** 2 + (max(beamy, 2527 - beamy)) ** 2) ** 0.5
)
testhighres = round(
wavelength / (2 * math.sin(0.5 * math.atan(largestvector / detsamdis))), 2
)
lowres = 50.0
highres = testhighres
datafilename = prefix + "_" + str(runno) + "_????"
mdatafilename = prefix + "_" + str(runno) + "_####.cbf"
seconds = 5 * exp_time
if angle_increment < 1 and not angle_increment == 0:
seconds = 5 * exp_time / angle_increment
# DEFINE SG/UNIT CELL
spacegroupnumber = ""
unitcellconstants = ""
datapath_dir = os.path.abspath(xds_file).replace("PROCESS_DATA", "RAW_DATA")
datapath_dir = os.path.dirname(os.path.dirname(datapath_dir)) + os.path.sep
# CREATE XDS.INP FILE
xds_templ = open(xds_template_path, "r").read()
xds_templ = xds_templ.replace("###BEAMX###", str(round(beamx, 2)))
xds_templ = xds_templ.replace("###BEAMY###", str(round(beamy, 2)))
xds_templ = xds_templ.replace("###DETSAMDIS###", str(round(detsamdis, 2)))
xds_templ = xds_templ.replace("###ANGLEINCREMENT###", str(angle_increment))
xds_templ = xds_templ.replace("###WAVELENGTH###", str(wavelength))
xds_templ = xds_templ.replace("###DATARANGESTARTNUM###", str(datarangestartnum))
xds_templ = xds_templ.replace(
"###DATARANGEFINISHNUM###", str(datarangefinishnum)
)
xds_templ = xds_templ.replace(
"###BACKGROUNDRANGESTART###", str(backgroundrangestartnum)
)
xds_templ = xds_templ.replace(
"###BACKGROUNDRANGEFINISHNUM###", str(backgroundrangefinishnum)
)
xds_templ = xds_templ.replace("###SPOTRANGESTARTNUM###", str(spotrangestartnum))
xds_templ = xds_templ.replace(
"###SPOTRANGEFINISHNUM###", str(spotrangefinishnum)
)
xds_templ = xds_templ.replace("###TESTLOWRES###", str(testlowres))
xds_templ = xds_templ.replace("###TESTHIGHRES###", str(testhighres))
xds_templ = xds_templ.replace("###LOWRES###", str(lowres))
xds_templ = xds_templ.replace("###HIGHRES###", str(highres))
xds_templ = xds_templ.replace("###DIRECTORY###", str(datapath_dir))
xds_templ = xds_templ.replace("###FILENAME###", str(datafilename))
xds_templ = xds_templ.replace("###SECONDS###", str(int(seconds)))
xds_templ = xds_templ.replace(
"###LYSOZYME_SPACE_GROUP_NUMBER###", str(spacegroupnumber)
)
xds_templ = xds_templ.replace(
"###LYSOZYME_UNIT_CELL_CONSTANTS###", str(unitcellconstants)
)
open(xds_file, "w").write(xds_templ)
# CREATE MOSFLM.DAT FILE
mosflm_templ = open(mosflm_template_path, "r").read()
mosflm_templ = mosflm_templ.replace("###DETSAMDIS###", str(round(detsamdis, 2)))
mosflm_templ = mosflm_templ.replace("###BEAMX###", str(round(mbeamx, 2)))
mosflm_templ = mosflm_templ.replace("###BEAMY###", str(round(mbeamy, 2)))
mosflm_templ = mosflm_templ.replace("###DIRECTORY###", str(datapath_dir))
mosflm_templ = mosflm_templ.replace("###FILENAME###", str(mdatafilename))
mosflm_templ = mosflm_templ.replace("###WAVELENGTH###", str(wavelength))
mosflm_templ = mosflm_templ.replace(
"###DATARANGESTARTNUM###", str(datarangestartnum)
)
open(mosflm_file, "w").write(mosflm_templ)
# CREATE EDNAPROC XML FILE
collection_id = dc_pars["collection_id"]
output_dir = dc_pars["ednaproc_dir"]
ednaproc_input_file = os.path.join(
output_dir, "EDNAprocInput_%d.xml" % collection_id
)
ednaproc_input = XSDataAutoprocInput()
input_file = XSDataFile()
path = XSDataString()
path.setValue(xds_file)
input_file.setPath(path)
ednaproc_input.setInput_file(input_file)
ednaproc_input.setData_collection_id(XSDataInteger(collection_id))
# output_dir = XSDataFile()
# outpath = XSDataString()
# outpath.setValue(output_dir)
# output_dir.setPath(path)
# ednaproc_input.setOutput_directory( output_dir )
ednaproc_input.exportToFile(ednaproc_input_file)
self.input_file = ednaproc_input_file
def createDataMOSFLMOutputIndexing(self):
self.DEBUG("EDPluginMOSFLMIndexingv10.createDataMOSFLMOutputIndexing")
xsDataMOSFLMOutputIndexing = XSDataMOSFLMOutputIndexing()
# Read Newmat file
xsDataMOSFLMNewmat = self.getDataMOSFLMNewmat()
if (xsDataMOSFLMNewmat is None):
strError = "MOSFLM indexing error : No solution was obtained!"
self.ERROR(strError)
self.setFailure()
else:
xsDataMOSFLMOutputIndexing.setRefinedNewmat(xsDataMOSFLMNewmat)
# Then read the XML file
strDnaTablesXML = self.readProcessFile(self.getScriptBaseName() +
"_dnaTables.xml")
xsDataDnaTables = dna_tables.parseString(strDnaTablesXML)
listXSTableMosaicityEstimation = EDUtilsTable.getTableListFromTables(
xsDataDnaTables, "mosaicity_estimation")
dMosaicityValueSum = 0.0
nValues = 0
for xsTableMosaicityEstimation in listXSTableMosaicityEstimation:
for xsListMosaicity in EDUtilsTable.getListsFromTable(
xsTableMosaicityEstimation, "mosaicity"):
dMosaicityValue = float(
EDUtilsTable.getItemFromList(xsListMosaicity,
"value").getValueOf_())
dMosaicityValueSum += dMosaicityValue
nValues += 1
xsDataFloatMosaicityEstimation = XSDataFloat()
xsDataFloatMosaicityEstimation.setValue(dMosaicityValueSum /
nValues)
xsDataMOSFLMOutputIndexing.setMosaicityEstimation(
xsDataFloatMosaicityEstimation)
xsTableRefinement = EDUtilsTable.getTableFromTables(
xsDataDnaTables, "refinement")
if (xsTableRefinement is None):
strError = "MOSFLM indexing error : No solution was refined!"
self.ERROR(strError)
self.setFailure()
else:
xsListDeviations = EDUtilsTable.getListsFromTable(
xsTableRefinement, "deviations")[0]
dDeviationAngular = float(
EDUtilsTable.getItemFromList(xsListDeviations,
"angular").getValueOf_())
dDeviationPositional = float(
EDUtilsTable.getItemFromList(xsListDeviations,
"positional").getValueOf_())
xsDataMOSFLMOutputIndexing.setDeviationAngular(
XSDataAngle(dDeviationAngular))
xsDataMOSFLMOutputIndexing.setDeviationPositional(
XSDataLength(dDeviationPositional))
xsListResults = EDUtilsTable.getListsFromTable(
xsTableRefinement, "results")[0]
dDetectorDistance = float(
EDUtilsTable.getItemFromList(
xsListResults, "detector_distance").getValueOf_())
xsDataMOSFLMOutputIndexing.setRefinedDistance(
XSDataLength(dDetectorDistance))
xsListParameters = EDUtilsTable.getListsFromTable(
xsTableRefinement, "parameters")[0]
iSpotsUsed = int(
EDUtilsTable.getItemFromList(xsListParameters,
"used").getValueOf_())
iSpotsTotal = int(
EDUtilsTable.getItemFromList(xsListParameters,
"out_of").getValueOf_())
xsDataMOSFLMOutputIndexing.setSpotsUsed(
XSDataInteger(iSpotsUsed))
xsDataMOSFLMOutputIndexing.setSpotsTotal(
XSDataInteger(iSpotsTotal))
xsTableSolutionRefinement = EDUtilsTable.getTableFromTables(
xsDataDnaTables, "solution_refinement")
xsListParameters = EDUtilsTable.getListsFromTable(
xsTableSolutionRefinement, "selection")[0]
iSelectedSolutionNumber = int(
EDUtilsTable.getItemFromList(xsListParameters,
"number").getValueOf_())
xsDataMOSFLMOutputIndexing.setSelectedSolutionNumber(
XSDataInteger(iSelectedSolutionNumber))
strSelectedSolutionSpaceGroup = (EDUtilsTable.getItemFromList(
xsListParameters, "spacegroup").getValueOf_())
xsDataMOSFLMOutputIndexing.setSelectedSolutionSpaceGroup(
XSDataString(strSelectedSolutionSpaceGroup))
iSelectedSolutionSpaceGroupNumber = int(
EDUtilsTable.getItemFromList(
xsListParameters, "spacegroup_number").getValueOf_())
xsDataMOSFLMOutputIndexing.setSelectedSolutionSpaceGroupNumber(
XSDataInteger(iSelectedSolutionSpaceGroupNumber))
xsTableAutoIndexSolutions = EDUtilsTable.getTableFromTables(
xsDataDnaTables, "autoindex_solutions")
xsListsSolution = xsTableAutoIndexSolutions.getList()
for xsListSolution in xsListsSolution:
xsDataMOSFLMIndexingSolution = XSDataMOSFLMIndexingSolution(
)
xsDataCell = XSDataCell()
iSolutionNumber = int(
EDUtilsTable.getItemFromList(xsListSolution,
"index").getValueOf_())
xsDataMOSFLMIndexingSolution.setIndex(
XSDataInteger(iSolutionNumber))
iPenalty = int(
EDUtilsTable.getItemFromList(xsListSolution,
"penalty").getValueOf_())
xsDataMOSFLMIndexingSolution.setPenalty(
XSDataInteger(iPenalty))
strLattice = (EDUtilsTable.getItemFromList(
xsListSolution, "lattice").getValueOf_())
xsDataMOSFLMIndexingSolution.setLattice(
XSDataString(strLattice))
dA = float(
EDUtilsTable.getItemFromList(xsListSolution,
"a").getValueOf_())
xsDataCell.setLength_a(XSDataLength(dA))
dB = float(
EDUtilsTable.getItemFromList(xsListSolution,
"b").getValueOf_())
xsDataCell.setLength_b(XSDataLength(dB))
dC = float(
EDUtilsTable.getItemFromList(xsListSolution,
"c").getValueOf_())
xsDataCell.setLength_c(XSDataLength(dC))
dAlpha = float(
EDUtilsTable.getItemFromList(xsListSolution,
"alpha").getValueOf_())
xsDataCell.setAngle_alpha(XSDataAngle(dAlpha))
dBeta = float(
EDUtilsTable.getItemFromList(xsListSolution,
"beta").getValueOf_())
xsDataCell.setAngle_beta(XSDataAngle(dBeta))
dGamma = float(
EDUtilsTable.getItemFromList(xsListSolution,
"gamma").getValueOf_())
xsDataCell.setAngle_gamma(XSDataAngle(dGamma))
xsDataMOSFLMIndexingSolution.setCell(xsDataCell)
xsDataMOSFLMOutputIndexing.addPossibleSolutions(
xsDataMOSFLMIndexingSolution)
xsTableBeamRefinement = EDUtilsTable.getTableFromTables(
xsDataDnaTables, "beam_refinement")
xsDataMOSFLMBeamPositionRefined = XSDataMOSFLMBeamPosition()
xsDataMOSFLMBeamPositionShift = XSDataMOSFLMBeamPosition()
dInitialBeamX = 0.0
dInitialBeamY = 0.0
dRefinedBeamX = 0.0
dRefinedBeamY = 0.0
xsListInitialBeam = EDUtilsTable.getListsFromTable(
xsTableBeamRefinement, "initial_beam")[0]
fInitialBeamPositionX = float(
EDUtilsTable.getItemFromList(xsListInitialBeam,
"x").getValueOf_())
fInitialBeamPositionY = float(
EDUtilsTable.getItemFromList(xsListInitialBeam,
"y").getValueOf_())
xsListRefinedBeam = EDUtilsTable.getListsFromTable(
xsTableBeamRefinement, "refined_beam")[0]
fRefinedBeamPositionX = float(
EDUtilsTable.getItemFromList(xsListRefinedBeam,
"x").getValueOf_())
fRefinedBeamPositionY = float(
EDUtilsTable.getItemFromList(xsListRefinedBeam,
"y").getValueOf_())
xsDataMOSFLMBeamPositionRefined.setX(
XSDataLength(fRefinedBeamPositionX))
xsDataMOSFLMBeamPositionRefined.setY(
XSDataLength(fRefinedBeamPositionY))
xsDataMOSFLMBeamPositionShift.setX(
XSDataLength(fInitialBeamPositionX -
fRefinedBeamPositionX))
xsDataMOSFLMBeamPositionShift.setY(
XSDataLength(fInitialBeamPositionY -
fRefinedBeamPositionY))
xsDataMOSFLMOutputIndexing.setRefinedBeam(
xsDataMOSFLMBeamPositionRefined)
xsDataMOSFLMOutputIndexing.setBeamShift(
xsDataMOSFLMBeamPositionShift)
# Path to log file
xsDataMOSFLMOutputIndexing.setPathToLogFile(
XSDataFile(
XSDataString(
os.path.join(self.getWorkingDirectory(),
self.getScriptLogFileName()))))
return xsDataMOSFLMOutputIndexing
def generateXSDataIndexingResult(_xsDataLabelitScreenOutput, _xsDataLabelitMosflmScriptsOutput, \
_xsDataExperimentalCondition=None):
EDVerbose.DEBUG(
"EDHandlerXSDataLabelitv1_1.generateXSDataIndexingOutput")
iSelectedSolutionNumber = _xsDataLabelitScreenOutput.getSelectedSolutionNumber(
).getValue()
xsDataIndexingResult = XSDataIndexingResult()
xsDataIndexingSolutionSelected = None
for xsDataLabelitSolution in _xsDataLabelitScreenOutput.getLabelitScreenSolution(
):
xsDataCrystal = XSDataCrystal()
xsDataSpaceGroup = XSDataSpaceGroup()
edStringSpaceGroupName = EDUtilsSymmetry.getMinimumSymmetrySpaceGroupFromBravaisLattice(
xsDataLabelitSolution.getBravaisLattice().getValue())
xsDataSpaceGroup.setName(XSDataString(edStringSpaceGroupName))
xsDataCrystal.setSpaceGroup(xsDataSpaceGroup)
xsDataCrystal.setCell(xsDataLabelitSolution.getUnitCell())
xsDataIndexingSolution = XSDataIndexingSolution()
xsDataIndexingSolution.setCrystal(xsDataCrystal)
iIndex = xsDataLabelitSolution.getSolutionNumber().getValue()
xsDataIndexingSolution.setNumber(XSDataInteger(iIndex))
xsDataIndexingResult.addSolution(xsDataIndexingSolution)
if (iIndex == iSelectedSolutionNumber):
xsDataIndexingSolutionSelected = XSDataIndexingSolutionSelected(
)
xsDataIndexingSolutionSelected.setNumber(XSDataInteger(iIndex))
edStringSelectedSpaceGroupName = edStringSpaceGroupName
xsDataCellSelected = xsDataLabelitSolution.getUnitCell()
fRmsdSelected = xsDataLabelitSolution.getRmsd().getValue()
iNumberOfSpotsSelected = xsDataLabelitSolution.getNumberOfSpots(
).getValue()
xsDataCrystalSelected = XSDataCrystal()
xsDataSpaceGroupSelected = XSDataSpaceGroup()
xsDataSpaceGroupSelected.setName(
XSDataString(edStringSelectedSpaceGroupName))
#xsDataSpaceGroupSelected.setITNumber( XSDataInteger( iSelectedSpaceGroupNumber ) )
xsDataCrystalSelected.setSpaceGroup(xsDataSpaceGroupSelected)
xsDataCrystalSelected.setCell(xsDataCellSelected)
xsDataCrystalSelected.setMosaicity(
XSDataDouble(_xsDataLabelitScreenOutput.getMosaicity().getValue()))
xsDataIndexingSolutionSelected.setCrystal(xsDataCrystalSelected)
xsDataOrientation = XSDataOrientation()
xsDataOrientation.setMatrixA(
_xsDataLabelitMosflmScriptsOutput.getAMatrix())
xsDataOrientation.setMatrixU(
_xsDataLabelitMosflmScriptsOutput.getUMatrix())
xsDataIndexingSolutionSelected.setOrientation(xsDataOrientation)
xsDataStatisticsIndexing = XSDataStatisticsIndexing()
if (_xsDataExperimentalCondition is not None):
fBeamPositionXOrig = _xsDataExperimentalCondition.getDetector(
).getBeamPositionX().getValue()
fBeamPositionYOrig = _xsDataExperimentalCondition.getDetector(
).getBeamPositionY().getValue()
fBeamPositionXNew = _xsDataLabelitScreenOutput.getBeamCentreX(
).getValue()
fBeamPositionYNew = _xsDataLabelitScreenOutput.getBeamCentreY(
).getValue()
xsDataStatisticsIndexing.setBeamPositionShiftX(
XSDataLength(fBeamPositionXOrig - fBeamPositionXNew))
xsDataStatisticsIndexing.setBeamPositionShiftY(
XSDataLength(fBeamPositionYOrig - fBeamPositionYNew))
#xsDataStatisticsIndexing.setSpotDeviXSDataLength( dDistanceRefinedationAngular( XSDataAngle( dDeviationAngular ) )
xsDataStatisticsIndexing.setSpotDeviationPositional(
XSDataLength(fRmsdSelected))
xsDataStatisticsIndexing.setSpotsUsed(
XSDataInteger(iNumberOfSpotsSelected))
xsDataStatisticsIndexing.setSpotsTotal(
XSDataInteger(iNumberOfSpotsSelected))
xsDataIndexingSolutionSelected.setStatistics(xsDataStatisticsIndexing)
xsDataExperimentalConditionRefined = None
if (_xsDataExperimentalCondition is None):
xsDataExperimentalConditionRefined = XSDataExperimentalCondition()
else:
# Copy the incoming experimental condition
xmlExperimentalCondition = _xsDataExperimentalCondition.marshal()
xsDataExperimentalConditionRefined = XSDataExperimentalCondition.parseString(
xmlExperimentalCondition)
xsDataDetector = xsDataExperimentalConditionRefined.getDetector()
if (xsDataDetector is None):
xsDataDetector = XSDataDetector()
xsDataDetector.setBeamPositionX(
_xsDataLabelitScreenOutput.getBeamCentreX())
xsDataDetector.setBeamPositionY(
_xsDataLabelitScreenOutput.getBeamCentreY())
xsDataDetector.setDistance(_xsDataLabelitScreenOutput.getDistance())
xsDataExperimentalConditionRefined.setDetector(xsDataDetector)
xsDataIndexingSolutionSelected.setExperimentalConditionRefined(
xsDataExperimentalConditionRefined)
xsDataIndexingResult.setSelectedSolution(
xsDataIndexingSolutionSelected)
xsDataIndexingResult.setIndexingLogFile(
_xsDataLabelitScreenOutput.getPathToLogFile())
return xsDataIndexingResult
def process(self, _edObject=None):
"""
Stores the contents of the AutoProcContainer in ISPyB.
"""
EDPluginExec.process(self)
self.DEBUG("EDPluginISPyBStoreScreeningv1_4.process")
xsDataInputISPyBStoreScreening = self.getDataInput()
httpAuthenticatedToolsForAutoprocessingWebService1 = HttpAuthenticated(
username=self.strUserName, password=self.strPassWord)
clientToolsForBLSampleWebServiceWsdl = Client(
self.strToolsForBLSampleWebServiceWsdl,
transport=httpAuthenticatedToolsForAutoprocessingWebService1)
httpAuthenticatedToolsForAutoprocessingWebService2 = HttpAuthenticated(
username=self.strUserName, password=self.strPassWord)
clientToolsForScreeningEDNAWebServiceWsdl = Client(
self.strToolsForScreeningEDNAWebServiceWsdl,
transport=httpAuthenticatedToolsForAutoprocessingWebService2)
self.bContinue = True
# Data collection Id
if xsDataInputISPyBStoreScreening.screening.dataCollectionId is None:
xsDataISPyBImage = xsDataInputISPyBStoreScreening.image
if xsDataISPyBImage is not None:
httpAuthenticatedToolsForAutoprocessingWebService3 = HttpAuthenticated(
username=self.strUserName, password=self.strPassWord)
clientToolsForCollectionWebService = Client(
self.strToolsForCollectionWebServiceWsdl,
transport=httpAuthenticatedToolsForAutoprocessingWebService3
)
self.iDataCollectionId = self.findDataCollectionFromFileLocationAndFileName(
clientToolsForCollectionWebService,
xsDataISPyBImage.fileLocation.value,
xsDataISPyBImage.fileName.value)
if self.iDataCollectionId is None:
self.ERROR("Couldn't obtain data collection id!")
self.setFailure()
else:
xsDataInputISPyBStoreScreening.screening.dataCollectionId = XSDataInteger(
self.iDataCollectionId)
else:
self.iDataCollectionId = xsDataInputISPyBStoreScreening.screening.dataCollectionId.value
if not self.isFailure():
# DiffractionPlan
xsDataISPyBDiffractionPlan = xsDataInputISPyBStoreScreening.diffractionPlan
iDiffractionPlanId = self.storeOrUpdateDiffractionPlan(
clientToolsForBLSampleWebServiceWsdl,
xsDataISPyBDiffractionPlan)
if iDiffractionPlanId is None:
self.ERROR(
"Couldn't create entry for diffraction plan in ISPyB!")
self.setFailure()
self.bContinue = False
# Screening
xsDataISPyBScreening = xsDataInputISPyBStoreScreening.screening
self.iScreeningId = self.storeOrUpdateScreening(
clientToolsForScreeningEDNAWebServiceWsdl,
xsDataISPyBScreening, iDiffractionPlanId)
if self.iScreeningId is None:
self.ERROR("Couldn't create entry for screening in ISPyB!")
self.setFailure()
self.bContinue = False
# Screening Output Container
for xsDataISPyBScreeningOutputContainer in xsDataInputISPyBStoreScreening.screeningOutputContainer:
xsDataISPyBScreeningOutput = xsDataISPyBScreeningOutputContainer.screeningOutput
iScreeningOutputId = self.storeOrUpdateScreeningOutput(
clientToolsForScreeningEDNAWebServiceWsdl,
xsDataISPyBScreeningOutput, self.iScreeningId)
if iScreeningOutputId is None:
self.ERROR("Couldn't create entry for screening in ISPyB!")
self.setFailure()
self.bContinue = False
for xsDataISPyBScreeningOutputLattice in xsDataISPyBScreeningOutputContainer.screeningOutputLattice:
iScreeningOutputLatticeId = self.storeOrUpdateScreeningOutputLattice(
clientToolsForScreeningEDNAWebServiceWsdl,
xsDataISPyBScreeningOutputLattice, iScreeningOutputId)
if iScreeningOutputLatticeId is None:
self.ERROR(
"Couldn't create entry for screening lattice in ISPyB!"
)
self.setFailure()
self.bContinue = False
for xsDataISPyBScreeningStrategyContainer in xsDataISPyBScreeningOutputContainer.screeningStrategyContainer:
# Create an empty object of type XSDataISPyBScreeningStrategy
xsDataISPyBScreeningStrategy = XSDataISPyBScreeningStrategy(
)
iScreeningStrategyId = self.storeOrUpdateScreeningStrategy(
clientToolsForScreeningEDNAWebServiceWsdl,
xsDataISPyBScreeningStrategy, iScreeningOutputId)
if iScreeningStrategyId is None:
self.ERROR(
"Couldn't create entry for screening strategy in ISPyB!"
)
self.setFailure()
self.bContinue = False
for xsDataISPyBScreeningStrategyWedgeContainer in xsDataISPyBScreeningStrategyContainer.screeningStrategyWedgeContainer:
xsDataISPyBScreeningStrategyWedge = xsDataISPyBScreeningStrategyWedgeContainer.screeningStrategyWedge
iScreeningStrategyWedgeId = self.storeOrUpdateScreeningStrategyWedge(
clientToolsForScreeningEDNAWebServiceWsdl,
xsDataISPyBScreeningStrategyWedge,
iScreeningStrategyId)
if iScreeningStrategyWedgeId is None:
self.ERROR(
"Couldn't create entry for screening strategy in ISPyB!"
)
self.setFailure()
self.bContinue = False
for xsDataISPyBScreeningStrategySubWedge in xsDataISPyBScreeningStrategyWedgeContainer.screeningStrategySubWedge:
iScreeningStrategySubWedgeId = self.storeOrUpdateScreeningStrategySubWedge(
clientToolsForScreeningEDNAWebServiceWsdl,
xsDataISPyBScreeningStrategySubWedge,
iScreeningStrategyWedgeId)
if iScreeningStrategySubWedgeId is None:
self.ERROR(
"Couldn't create entry for screening strategy in ISPyB!"
)
self.setFailure()
self.bContinue = False
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG('EDPluginControlXDSAPPv1_0.process starting')
directory = None
template = None
imageNoStart = None
imageNoEnd = None
pathToStartImage = None
pathToEndImage = None
userName = os.environ["USER"]
beamline = "unknown"
proposal = "unknown"
if self.dataInput.startImageNumber is not None:
imageNoStart = self.dataInput.startImageNumber.value
if self.dataInput.endImageNumber is not None:
imageNoEnd = self.dataInput.endImageNumber.value
# If we have a data collection id, use it
if self.dataInput.dataCollectionId is not None:
# Recover the data collection from ISPyB
xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
identifier = str(self.dataInput.dataCollectionId.value)
xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
self.edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
self.edPluginRetrieveDataCollection.executeSynchronous()
ispybDataCollection = self.edPluginRetrieveDataCollection.dataOutput.dataCollection
directory = ispybDataCollection.imageDirectory
template = ispybDataCollection.fileTemplate.replace("%04d", "####")
if imageNoStart is None:
imageNoStart = ispybDataCollection.startImageNumber
if imageNoEnd is None:
imageNoEnd = ispybDataCollection.startImageNumber + \
ispybDataCollection.numberOfImages - 1
# # DEBUG we set the end image to 20 in order to speed up things
# self.warning("End image set to 20 (was {0})".format(imageNoEnd))
# imageNoEnd = 20
pathToStartImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, ispybDataCollection.fileTemplate % imageNoEnd)
else:
identifier = str(int(time.time()))
directory = self.dataInput.dirN.value
template = self.dataInput.templateN.value
fileTemplate = template.replace("####", "%04d")
pathToStartImage = os.path.join(directory, fileTemplate % imageNoStart)
pathToEndImage = os.path.join(directory, fileTemplate % imageNoEnd)
# Try to get proposal from path
if EDUtilsPath.isESRF():
listDirectory = directory.split(os.sep)
try:
if listDirectory[1] == "data":
if listDirectory[2] == "visitor":
beamline = listDirectory[4]
proposal = listDirectory[3]
else:
beamline = listDirectory[2]
proposal = listDirectory[4]
except:
beamline = "unknown"
proposal = userName
if imageNoEnd - imageNoStart < 8:
error_message = "There are fewer than 8 images, aborting"
self.addErrorMessage(error_message)
self.ERROR(error_message)
self.setFailure()
return
# Process directory
if self.dataInput.processDirectory is not None:
processDirectory = self.dataInput.processDirectory.path.value
else:
processDirectory = directory.replace("RAW_DATA", "PROCESSED_DATA")
# Make results directory
self.resultsDirectory = os.path.join(processDirectory, "results")
if not os.path.exists(self.resultsDirectory):
os.makedirs(self.resultsDirectory, 0o755)
# Create path to pyarch
if self.dataInput.reprocess is not None and self.dataInput.reprocess.value:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchReprocessDirectoryPath(beamline,
"XDSAPP", self.dataInput.dataCollectionId.value)
else:
self.pyarchDirectory = EDHandlerESRFPyarchv1_0.createPyarchFilePath(self.resultsDirectory)
self.pyarchDirectory = self.pyarchDirectory.replace('PROCESSED_DATA', 'RAW_DATA')
if self.pyarchDirectory is not None and not os.path.exists(self.pyarchDirectory):
try:
os.makedirs(self.pyarchDirectory, 0o755)
except:
self.pyarchDirectory = None
# Determine pyarch prefix
listPrefix = template.split("_")
self.pyarchPrefix = "xa_{0}_run{1}".format(listPrefix[-3], listPrefix[-2])
isH5 = False
if any(beamline in pathToStartImage for beamline in ["id30b"]):
minSizeFirst = 6000000
minSizeLast = 6000000
elif any(beamline in pathToStartImage for beamline in ["id23eh2", "id30a1"]):
minSizeFirst = 2000000
minSizeLast = 2000000
elif any(beamline in pathToStartImage for beamline in ["id23eh1", "id30a3"]):
minSizeFirst = 100000
minSizeLast = 100000
pathToStartImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoStart))
pathToEndImage = os.path.join(directory,
self.eiger_template_to_image(template, imageNoEnd))
isH5 = True
else:
minSizeFirst = 1000000
minSizeLast = 1000000
fWaitFileTimeout = 3600 # s
xsDataInputMXWaitFileFirst = XSDataInputMXWaitFile()
xsDataInputMXWaitFileFirst.file = XSDataFile(XSDataString(pathToStartImage))
xsDataInputMXWaitFileFirst.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileFirst.size = XSDataInteger(minSizeFirst)
self.edPluginWaitFileFirst.dataInput = xsDataInputMXWaitFileFirst
self.edPluginWaitFileFirst.executeSynchronous()
if self.edPluginWaitFileFirst.dataOutput.timedOut.value:
strWarningMessage = "Timeout after %d seconds waiting for the first image %s!" % (fWaitFileTimeout, pathToStartImage)
self.addWarningMessage(strWarningMessage)
self.WARNING(strWarningMessage)
xsDataInputMXWaitFileLast = XSDataInputMXWaitFile()
xsDataInputMXWaitFileLast.file = XSDataFile(XSDataString(pathToEndImage))
xsDataInputMXWaitFileLast.timeOut = XSDataTime(fWaitFileTimeout)
self.edPluginWaitFileLast.size = XSDataInteger(minSizeLast)
self.edPluginWaitFileLast.dataInput = xsDataInputMXWaitFileLast
self.edPluginWaitFileLast.executeSynchronous()
if self.edPluginWaitFileLast.dataOutput.timedOut.value:
strErrorMessage = "Timeout after %d seconds waiting for the last image %s!" % (fWaitFileTimeout, pathToEndImage)
self.addErrorMessage(strErrorMessage)
self.ERROR(strErrorMessage)
self.setFailure()
if self.dataInput.hdf5ToCbfDirectory is not None:
dir = self.dataInput.hdf5ToCbfDirectory.path.value
pathToStartImage = glob.glob(os.path.join(dir, "*000001*"))[0]
self.timeStart = time.localtime()
# Prepare input to execution plugin
if self.doAnom:
xsDataInputXDSAPPAnom = XSDataInputXDSAPP()
xsDataInputXDSAPPAnom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPAnom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPAnom.anomalous = XSDataBoolean(True)
xsDataInputXDSAPPAnom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPAnom.spacegroup = XSDataString(self.xdsAppSpacegroup)
self.edPluginExecXDSAPPAnom.dataInput = xsDataInputXDSAPPAnom
self.edPluginExecXDSAPPAnom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=True,
timeStart=self.timeStart)
if self.doNoanom:
xsDataInputXDSAPPNoanom = XSDataInputXDSAPP()
xsDataInputXDSAPPNoanom.startImageNumber = self.dataInput.startImageNumber
xsDataInputXDSAPPNoanom.endImageNumber = self.dataInput.endImageNumber
xsDataInputXDSAPPNoanom.anomalous = XSDataBoolean(False)
xsDataInputXDSAPPNoanom.image = XSDataFile(XSDataString(pathToStartImage))
if self.xdsAppSpacegroup is not None:
xsDataInputXDSAPPNoanom.spacegroup = XSDataString(self.xdsAppSpacegroup)
self.edPluginExecXDSAPPNoanom.dataInput = xsDataInputXDSAPPNoanom
self.edPluginExecXDSAPPNoanom.execute()
if self.dataInput.dataCollectionId is not None:
# Set ISPyB to started
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom = \
EDHandlerXSDataISPyBv1_4.setIspybToRunning(self, dataCollectionId=self.dataInput.dataCollectionId.value,
processingPrograms=self.processingPrograms, isAnom=False,
timeStart=self.timeStart)
if self.doAnom:
self.edPluginExecXDSAPPAnom.synchronize()
xsDataResultXDSAPPAnom = self.edPluginExecXDSAPPAnom.dataOutput
# Run XSCALE even if XSCALE.LP is present
strPathXscaleLpAnom = self.runXscale(self.edPluginExecXDSAPPAnom.getWorkingDirectory(), anom=True, merged=True)
if self.doNoanom:
self.edPluginExecXDSAPPNoanom.synchronize()
xsDataResultXDSAPPNoanom = self.edPluginExecXDSAPPNoanom.dataOutput
strPathXscaleLpNoanom = self.runXscale(self.edPluginExecXDSAPPNoanom.getWorkingDirectory(), anom=False, merged=True)
self.timeEnd = time.localtime()
# Upload to ISPyB
if self.dataInput.dataCollectionId is not None:
# Check if we should use XDS_ASCII_to_XML.pl
if self.doAnom:
if self.useXdsAsciiToXml:
# Only for anom runs
self.runXdsAsciiToXml(xsDataResultXDSAPPAnom,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
else:
self.hasUploadedAnomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPAnom, processDirectory, template,
strPathXscaleLpAnom, True, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdAnom, self.autoProcProgramIdAnom)
if self.hasUploadedAnomResultsToISPyB:
self.screen("Anom results uploaded to ISPyB")
else:
self.ERROR("Could not upload anom results to ISPyB!")
if self.doNoanom:
self.hasUploadedNoanomResultsToISPyB = self.uploadToISPyB(xsDataResultXDSAPPNoanom, processDirectory, template,
strPathXscaleLpNoanom, False, proposal, self.timeStart, self.timeEnd,
self.dataInput.dataCollectionId.value,
self.autoProcIntegrationIdNoanom, self.autoProcProgramIdNoanom)
if self.hasUploadedNoanomResultsToISPyB:
self.screen("Noanom results uploaded to ISPyB")
else:
self.ERROR("Could not upload noanom results to ISPyB!")
def getOutputDataFromDNATableFile(self, _strFileName):
xsDataResultBest = XSDataResultBest()
strDnaTablesXML = self.readProcessFile(_strFileName)
xsDataDnaTables = dna_tables.parseString(strDnaTablesXML)
# Loop through all the tables and fill in the relevant parts of xsDataResultBest
# SubWedges
xsTablesCollectionStrategy = EDUtilsTable.getTableListFromTables(
xsDataDnaTables, "data_collection_strategy")
iCollectionPlanNumber = 1
for xsTableCollectionStrategy in xsTablesCollectionStrategy:
xsDataBestCollectionPlan = XSDataBestCollectionPlan()
xsCollectionRunList = EDUtilsTable.getListsFromTable(
xsTableCollectionStrategy, "collection_run")
iCollectionRunNumber = 1
for xsCollectionRunItemList in xsCollectionRunList:
xsDataCollectionRun = self.collectionRunItemListToCollectionRun(
xsCollectionRunItemList)
xsDataCollectionRun.setCollectionRunNumber(
XSDataInteger(iCollectionRunNumber))
xsDataBestCollectionPlan.addCollectionRun(xsDataCollectionRun)
iCollectionRunNumber = iCollectionRunNumber + 1
# Strategy Summary
xsStrategySummaryItemList = EDUtilsTable.getListsFromTable(
xsTableCollectionStrategy, "summary")
xsDataStrategySummary = self.strategySummaryItemListToStrategySummary(
xsStrategySummaryItemList[0])
# Ranking Resolution
# Not part of strategySummaryItemListToStrategySummary method since it is in the general_form part
xsTableGeneralInform = EDUtilsTable.getTableFromTables(
xsDataDnaTables, "general_inform")
xsRankingResolutionItemList = EDUtilsTable.getListsFromTable(
xsTableGeneralInform, "ranking_resolution")
xsItemRankingResolution = EDUtilsTable.getItemFromList(
xsRankingResolutionItemList[0], "dmin")
fRankingResolution = float(xsItemRankingResolution.getValueOf_())
xsDataStrategySummary.setRankingResolution(
XSDataDouble(fRankingResolution))
xsDataBestCollectionPlan.setStrategySummary(xsDataStrategySummary)
# Satistics
xsTablesStatisticalPrediction = EDUtilsTable.getTableListFromTables(
xsDataDnaTables, "statistical_prediction")
for xsTableStatisticalPrediction in xsTablesStatisticalPrediction:
if (xsTableStatisticalPrediction.getIndex() ==
xsTableCollectionStrategy.getIndex()):
xsResolutionBinList = EDUtilsTable.getListsFromTable(
xsTableStatisticalPrediction, "resolution_bin")
xsDataStatisticalPrediction = XSDataBestStatisticalPrediction(
)
for xsResolutionBinItemList in xsResolutionBinList:
xsDataResolutionBin = self.resolutionBinItemListToResolutionBin(
xsResolutionBinItemList)
xsDataStatisticalPrediction.addResolutionBin(
xsDataResolutionBin)
xsDataBestCollectionPlan.setStatisticalPrediction(
xsDataStatisticalPrediction)
xsDataBestCollectionPlan.setCollectionPlanNumber(
XSDataInteger(iCollectionPlanNumber))
xsDataResultBest.addCollectionPlan(xsDataBestCollectionPlan)
iCollectionPlanNumber = iCollectionPlanNumber + 1
# Fix the order of the collection plans - then low resolution pass should be the first one
listCollectionPlan = xsDataResultBest.getCollectionPlan()
if (len(listCollectionPlan) > 1):
bIsModified = False
for xsDataCollectionPlan in listCollectionPlan:
xsDataStrategySummary = xsDataCollectionPlan.getStrategySummary(
)
strReasoning = xsDataStrategySummary.getResolutionReasoning(
).getValue()
if (strReasoning.find("Low-resolution") !=
-1) and (xsDataCollectionPlan.getCollectionPlanNumber(
).getValue() != 1):
listCollectionPlan.remove(xsDataCollectionPlan)
listCollectionPlan.insert(0, xsDataCollectionPlan)
bIsModified = True
if (bIsModified):
iCollectionPlanNumber = 1
for xsDataCollectionPlan in listCollectionPlan:
xsDataCollectionPlan.setCollectionPlanNumber(
XSDataInteger(iCollectionPlanNumber))
iCollectionPlanNumber = iCollectionPlanNumber + 1
return xsDataResultBest
def create_autoproc_input(self, event, params):
"""
Descript. :
"""
WAIT_XDS_TIMEOUT = 20
WAIT_XDS_RESOLUTION = 1
file_name_timestamp = time.strftime("%Y%m%d_%H%M%S")
autoproc_path = params.get("xds_dir")
autoproc_xds_filename = os.path.join(autoproc_path, "XDS.INP")
autoproc_input_filename = os.path.join(autoproc_path,
"edna-autoproc-input-%s" % \
file_name_timestamp)
autoproc_output_file_name = os.path.join(autoproc_path,
"edna-autoproc-results-%s" % \
file_name_timestamp)
autoproc_input = XSDataAutoprocInput()
autoproc_xds_file = XSDataFile()
autoproc_xds_file.setPath(XSDataString(autoproc_xds_filename))
autoproc_input.setInput_file(autoproc_xds_file)
autoproc_output_file = XSDataFile()
autoproc_output_file.setPath(XSDataString(autoproc_output_file_name))
autoproc_input.setOutput_file(autoproc_output_file)
autoproc_input.setData_collection_id(
XSDataInteger(params.get("collection_id")))
residues_num = float(params.get("residues", 0))
if residues_num != 0:
autoproc_input.setNres(XSDataDouble(residues_num))
space_group = params.get("sample_reference").get("spacegroup", "")
if len(space_group) > 0:
autoproc_input.setSpacegroup(XSDataString(space_group))
unit_cell = params.get("sample_reference").get("cell", "")
if len(unit_cell) > 0:
autoproc_input.setUnit_cell(XSDataString(unit_cell))
#Maybe we have to check if directory is there. Maybe create dir with mxcube
xds_appeared = False
wait_xds_start = time.time()
logging.debug('EMBLAutoprocessing: Waiting for XDS.INP file: %s' %
autoproc_xds_filename)
while not xds_appeared and time.time(
) - wait_xds_start < WAIT_XDS_TIMEOUT:
if os.path.exists(autoproc_xds_filename
) and os.stat(autoproc_xds_filename).st_size > 0:
xds_appeared = True
logging.debug('EMBLAutoprocessing: XDS.INP file is there, size={0}'.\
format(os.stat(autoproc_xds_filename).st_size))
else:
os.system("ls %s> /dev/null" %
(os.path.dirname(autoproc_path)))
gevent.sleep(WAIT_XDS_RESOLUTION)
if not xds_appeared:
logging.error('EMBLAutoprocessing: XDS.INP file ({0}) failed to appear after {1} seconds'.\
format(autoproc_xds_filename, WAIT_XDS_TIMEOUT))
return None, False
autoproc_input.exportToFile(autoproc_input_filename)
return autoproc_input_filename, True
q = datapoint[:, 0]
I = datapoint[:, 1]
s = datapoint[:, 2]
if qMax is not None:
mask = (q < qMax)
q = q[mask]
I = I[mask]
s = s[mask]
xsd = XSDataInputSolutionScattering(
experimentalDataQArray=EDUtilsArray.arrayToXSData(
q / 10.), #from nm-1 not A-1
experimentalDataIArray=EDUtilsArray.arrayToXSData(I),
experimentalDataStdArray=EDUtilsArray.arrayToXSData(s),
title=XSDataString(title.strip()),
angularUnits=XSDataInteger(1))
if self.__class__.maxThreads is not None:
xsd.iNbThreads = XSDataInteger(self.__class__.maxThreads)
self.__edPluginExecSAS = self.loadPlugin(self.__strControlledPluginSAS)
self.__edPluginExecSAS.dataInput = xsd
self.__edPluginExecSAS.connectSUCCESS(self.doSuccessExecSAS)
self.__edPluginExecSAS.connectFAILURE(self.doFailureExecSAS)
self.__edPluginExecSAS.executeSynchronous()
if self.isFailure():
return
if self.dataInput.destinationDirectory is None:
outdir = os.path.join(
os.path.dirname(os.path.dirname(self.strInFile)), "ednaSAS")
else:
outdir = self.dataInput.destinationDirectory.path.value
