def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXdsFastProc.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.second_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG('copying previously generated files to the new plugin dir')
copy_xds_files(self.first_run.getWorkingDirectory(),
self.second_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
# increase all the spot ranges end by 20, constrained to
# the data range upper limit
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 20
if range_end > self.end_image_no:
self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.third_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG('copying previously generated files to the new plugin dir')
copy_xds_files(self.second_run.getWorkingDirectory(),
self.third_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 40
if range_end > self.end_image_no:
self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXds.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# second run w/ JOB set to DEFPIX INTEGRATE CORRECT
self.second_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1
self.third_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'ALL'
params.maxprocs = 4
params.maxjobs = 1
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# final run with parallelism like 3 but JOB like 2
self.final_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
params.maxprocs = 4
params.maxjobs = 1
self.final_run.dataInput = params
self.final_run.executeSynchronous()
EDVerbose.DEBUG('final run completed')
if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.final_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXdsFastProc.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.second_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG(
'copying previously generated files to the new plugin dir')
copy_xds_files(self.first_run.getWorkingDirectory(),
self.second_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
# increase all the spot ranges end by 20, constrained to
# the data range upper limit
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 20
if range_end > self.end_image_no:
self.DEBUG(
'End of range {0} would be past the last image {1}'.
format(range_end, self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.third_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG(
'copying previously generated files to the new plugin dir')
copy_xds_files(self.second_run.getWorkingDirectory(),
self.third_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 40
if range_end > self.end_image_no:
self.DEBUG(
'End of range {0} would be past the last image {1}'.
format(range_end, self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXds.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# second run w/ JOB set to DEFPIX INTEGRATE CORRECT
self.second_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1
self.third_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'ALL'
params.maxprocs = 4
params.maxjobs = 1
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# final run with parallelism like 3 but JOB like 2
self.final_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
params.maxprocs = 4
params.maxjobs = 1
self.final_run.dataInput = params
self.final_run.executeSynchronous()
EDVerbose.DEBUG('final run completed')
if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.final_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlAutoproc.process")
# first XDS plugin run with supplied XDS file
EDVerbose.screen("First XDS run...")
self.xds_first.executeSynchronous()
if self.xds_first.isFailure():
EDVerbose.ERROR('first XDS run failed')
self.setFailure()
return
EDVerbose.screen("First XDS run successfully completed")
# apply the first res cutoff with the res extracted from the first XDS run
EDVerbose.screen("First resolution cutoff")
xdsresult = self.xds_first.dataOutput
res_cutoff_in = XSDataResCutoff()
res_cutoff_in.xds_res = xdsresult
res_cutoff_in.completeness_entries = xdsresult.completeness_entries
#XXX: remove from the data model as it is just pass-through?
res_cutoff_in.total_completeness = xdsresult.total_completeness
self.first_res_cutoff.dataInput = res_cutoff_in
self.first_res_cutoff.executeSynchronous()
if self.first_res_cutoff.isFailure():
EDVerbose.ERROR("res cutoff failed")
self.setFailure()
return
EDVerbose.screen("First resolution cutoff successfully completed")
resolution = self.first_res_cutoff.dataOutput.res
# for the generate w/ and w/out anom we have to specify where
# the first XDS plugin run took place
xds_run_directory = os.path.abspath(self.xds_first.dataOutput.xds_run_directory.value)
EDVerbose.screen("the xds run took place in {}".format(xds_run_directory))
generate_input = XSDataXdsGenerateInput()
generate_input.resolution = resolution
generate_input.previous_run_dir = XSDataString(xds_run_directory)
self.generate.dataInput = generate_input
self.generate.executeSynchronous()
if self.generate.isFailure():
EDVerbose.ERROR('generating w/ and w/out anom failed')
self.setFailure()
return
# we can now use the xds output parser on the two correct.lp
# files, w/ and w/out anom
parse_anom_input = XSDataXdsOutputFile()
parse_anom_input.correct_lp = XSDataFile()
parse_anom_input.correct_lp.path = self.generate.dataOutput.correct_lp_anom
self.parse_xds_anom.dataInput = parse_anom_input
self.parse_xds_anom.executeSynchronous()
if self.parse_xds_anom.isFailure():
EDVerbose.ERROR('parsing the xds generated w/ anom failed')
self.setFailure()
return
# now the other one w/out anom
parse_noanom_input = XSDataXdsOutputFile()
parse_noanom_input.correct_lp = XSDataFile()
parse_noanom_input.correct_lp.path = self.generate.dataOutput.correct_lp_no_anom
self.parse_xds_noanom.dataInput = parse_noanom_input
self.parse_xds_noanom.executeSynchronous()
if self.parse_xds_noanom.isFailure():
EDVerbose.ERROR('parsing the xds generated w/ anom failed')
self.setFailure()
return
# we now can apply the res cutoff on the anom and no anom
# outputs. Note that this is not done in parallel, like the
# xds parsing
# XXX completeness_cutoff/res_override and isig_cutoff still
# missing
res_cutoff_anom_in = XSDataResCutoff()
res_cutoff_anom_in.xds_res = self.parse_xds_anom.dataOutput
self.res_cutoff_anom.dataInput = res_cutoff_anom_in
self.res_cutoff_anom.executeSynchronous()
if self.res_cutoff_anom.isFailure():
EDVerbose.ERROR('res cutoff for anom data failed')
# same for non anom
res_cutoff_noanom_in = XSDataResCutoff()
res_cutoff_noanom_in.xds_res = self.parse_xds_noanom.dataOutput
self.res_cutoff_noanom.dataInput = res_cutoff_noanom_in
self.res_cutoff_noanom.executeSynchronous()
if self.res_cutoff_noanom.isFailure():
EDVerbose.ERROR('res cutoff for non anom data failed')
# now we just have to run XScale to generate w/ and w/out
# anom, merged and unmerged
# We use another control plugin for that to isolate the whole thing
# if not self.res_cutoff_anom.isFailure():
xscale_anom_in = XSDataXscaleInput()
input_file = XSDataXscaleInputFile()
input_file.path = self.generate.dataOutput.hkl_anom
input_file.res = self.res_cutoff_anom.dataOutput.res
xscale_anom_in.xds_files = [input_file]
xscale_anom_in.unit_cell_constants = self.parse_xds_anom.dataOutput.unit_cell_constants
xscale_anom_in.sg_number = self.parse_xds_anom.dataOutput.sg_number
xscale_anom_in.bins = self.res_cutoff_anom.bins
self.xscale_anom.dataInput = xscale_anom_in
self.xscale_anom.executeSynchronous()
if self.xscale_anom.isFailure():
EDVerbose.ERROR('xscale anom/merge generation failed')
# same for non anom code path
#if not self.res_cutoff_noanom.isFailure():
xscale_noanom_in = XSDataXscaleInput()
input_file = XSDataXscaleInputFile()
input_file.path = self.generate.dataOuput.hkl_anom
input_file.res = self.res_cutoff_anom.dataOutput.res
xscale_noanom_in.xds_files = [input_file]
xscale_noanom_in.unit_cell_constants = self.parse_xds_noanom.dataOutput.unit_cell_constants
xscale_noanom_in.sg_number = self.parse_xds_noanom.dataOutput.sg_number
xscale_noanom_in.bins = self.res_cutoff_noanom.bins
self.xscale_noanom.dataInput = xscale_noanom_in
self.xscale_noanom.executeSynchronous()
if self.xscale_noanom.isFailure():
EDVerbose.ERROR('xscale anom/merge generation failed')
log_to_ispyb([self.integration_id_anom, self.integration_id_noanom],
'Scaling',
'Failed',
'anom/noanom generation failed in {0}s'.format(self.stats['anom/noanom_generation']))
return
else:
EDVerbose.screen('generating w/ and w/out anom finished')
log_to_ispyb([self.integration_id_anom, self.integration_id_noanom],
'Scaling',
'Successful',
'anom/noanom generation finished in {0}s'.format(self.stats['anom/noanom_generation']))
# we can now use the xds output parser on the two correct.lp
# files, w/ and w/out anom
parse_anom_input = XSDataXdsOutputFile()
parse_anom_input.correct_lp = XSDataFile()
parse_anom_input.correct_lp.path = self.generate.dataOutput.correct_lp_anom
# this one is the same as the first XDS run since they share
# the same directory
gxparm_file_anom = XSDataFile()
gxparm_file_anom.path = self.generate.dataOutput.gxparm
parse_anom_input.gxparm = gxparm_file_anom
self.parse_xds_anom.dataInput = parse_anom_input
self.parse_xds_anom.executeSynchronous()
if self.parse_xds_anom.isFailure():
EDVerbose.ERROR('parsing the xds generated w/ anom failed')
self.setFailure()
'Scaling', 'Failed',
'anom/noanom generation failed in {0}s'.format(
self.stats['anom/noanom_generation']))
return
else:
EDVerbose.screen('generating w/ and w/out anom finished')
log_to_ispyb(
[self.integration_id_anom, self.integration_id_noanom],
'Scaling', 'Successful',
'anom/noanom generation finished in {0}s'.format(
self.stats['anom/noanom_generation']))
# we can now use the xds output parser on the two correct.lp
# files, w/ and w/out anom
parse_anom_input = XSDataXdsOutputFile()
parse_anom_input.correct_lp = XSDataFile()
parse_anom_input.correct_lp.path = self.generate.dataOutput.correct_lp_anom
# this one is the same as the first XDS run since they share
# the same directory
gxparm_file_anom = XSDataFile()
gxparm_file_anom.path = self.generate.dataOutput.gxparm
parse_anom_input.gxparm = gxparm_file_anom
self.parse_xds_anom.dataInput = parse_anom_input
self.parse_xds_anom.executeSynchronous()
if self.parse_xds_anom.isFailure():
EDVerbose.ERROR('parsing the xds generated w/ anom failed')
self.setFailure()