def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXdsFastProc.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.second_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG('copying previously generated files to the new plugin dir')
copy_xds_files(self.first_run.getWorkingDirectory(),
self.second_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
# increase all the spot ranges end by 20, constrained to
# the data range upper limit
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 20
if range_end > self.end_image_no:
self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.third_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG('copying previously generated files to the new plugin dir')
copy_xds_files(self.second_run.getWorkingDirectory(),
self.third_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 40
if range_end > self.end_image_no:
self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG('EDPluginControlAutoproc.preProcess starting')
self.DEBUG('failure state is currently {0}'.format(self.isFailure()))
# for info to send to the autoproc stats server
self.custom_stats = dict(creation_time=time.time(),
processing_type='edna fastproc',
datacollect_id=self.dataInput.data_collection_id.value,
comments='running on {0}'.format(socket.gethostname()))
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
xds_in.spacegroup = data_in.spacegroup
xds_in.unit_cell = data_in.unit_cell
self.log_file_path = os.path.join(self.root_dir, 'stats.json')
self.DEBUG('will log timing information to {0}'.format(self.log_file_path))
self.stats = dict()
# Get the image prefix from the directory name
# XXX: This is horrible
try:
self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1])
except Exception:
self.image_prefix = ''
self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing')
try:
os.makedirs(self.results_dir)
except OSError: # it most likely exists
pass
# Copy the vanilla XDS input file to the results dir
infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP')
shutil.copy(self.dataInput.input_file.path.value,
infile_dest)
# Ensure the autoproc ids directory is there
self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids')
try:
os.makedirs(self.autoproc_ids_dir)
except OSError: # it's there
pass
# we'll need the low res limit later on
lowres = data_in.low_resolution_limit
if lowres is not None:
self.low_resolution_limit = lowres.value
else:
self.low_resolution_limit = 50
res_override = data_in.res_override
if res_override is not None:
self.res_override = res_override.value
else:
# XXX: default to 0?
self.res_override = None
# check the number of images (must be > 8) and get the first
# image name to wait for. Also modify the XDS.INP file to
# reflect these values, if specified
conf = parse_xds_file(data_in.input_file.path.value)
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in conf:
xgeo = os.path.abspath(os.path.join(self.root_dir,
conf['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
del conf['X-GEO_CORR=']
else:
conf['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in conf:
ygeo = os.path.abspath(os.path.join(self.root_dir,
conf['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
del conf['Y-GEO_CORR=']
else:
conf['Y-GEO_CORR='] = ygeo
dump_xds_file(data_in.input_file.path.value, conf)
resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')
if resrange is not None:
if self.low_resolution_limit is not None:
resrange[0] = self.low_resolution_limit
if self.res_override is not None:
resrange[1] = self.res_override
conf['INCLUDE_RESOLUTION_RANGE='] = resrange
dump_xds_file(data_in.input_file.path.value, conf)
data_range = conf.get('DATA_RANGE=')
# we'll need that for the very last part ( file import )
self.data_range = data_range
if data_range is not None:
start_image = data_range[0]
end_image = data_range[1]
if end_image - start_image < 8:
self.ERROR('there are fewer than 8 images, aborting')
self.setFailure()
return
template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
self.DEBUG('template for images is {0}'.format(template))
# fix the path if it's not absolute
if not os.path.isabs(template):
self.DEBUG('file template {0} is not absolute'.format(template))
base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value))
template = os.path.normpath(os.path.join(self.root_dir, template))
conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template
self.DEBUG('file template fixed to {0}'.format(template))
self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value))
dump_xds_file(data_in.input_file.path.value, conf)
first_image = _template_to_image(template, start_image)
self.wait_file = self.loadPlugin('EDPluginWaitFile')
waitfileinput = XSDataInputWaitFile()
waitfileinput.expectedFile = XSDataFile()
waitfileinput.expectedFile.path = XSDataString(first_image)
waitfileinput.expectedSize = XSDataInteger(0) # we do not care
timeout = XSDataTime()
global WAIT_FOR_FRAME_TIMEOUT
timeout.value = WAIT_FOR_FRAME_TIMEOUT
waitfileinput.timeOut = timeout
self.wait_file.dataInput = waitfileinput
self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")
self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')
self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXdsFastProc.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.second_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG(
'copying previously generated files to the new plugin dir')
copy_xds_files(self.first_run.getWorkingDirectory(),
self.second_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
# increase all the spot ranges end by 20, constrained to
# the data range upper limit
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 20
if range_end > self.end_image_no:
self.DEBUG(
'End of range {0} would be past the last image {1}'.
format(range_end, self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.third_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG(
'copying previously generated files to the new plugin dir')
copy_xds_files(self.second_run.getWorkingDirectory(),
self.third_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 40
if range_end > self.end_image_no:
self.DEBUG(
'End of range {0} would be past the last image {1}'.
format(range_end, self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG('EDPluginControlAutoproc.preProcess starting')
self.DEBUG('failure state is currently {0}'.format(self.isFailure()))
# for info to send to the autoproc stats server
self.custom_stats = dict(
creation_time=time.time(),
processing_type='edna fastproc',
datacollect_id=self.dataInput.data_collection_id.value,
comments='running on {0}'.format(socket.gethostname()))
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
xds_in.spacegroup = data_in.spacegroup
xds_in.unit_cell = data_in.unit_cell
self.log_file_path = os.path.join(self.root_dir, 'stats.json')
self.DEBUG('will log timing information to {0}'.format(
self.log_file_path))
self.stats = dict()
# Get the image prefix from the directory name
# XXX: This is horrible
try:
self.image_prefix = '_'.join(
os.path.basename(self.root_dir).split('_')[1:-1])
except Exception:
self.image_prefix = ''
self.results_dir = os.path.join(self.root_dir, 'results',
'fast_processing')
try:
os.makedirs(self.results_dir)
except OSError: # it most likely exists
pass
# Copy the vanilla XDS input file to the results dir
infile_dest = os.path.join(self.results_dir,
self.image_prefix + '_input_XDS.INP')
shutil.copy(self.dataInput.input_file.path.value, infile_dest)
# Ensure the autoproc ids directory is there
self.autoproc_ids_dir = os.path.join(self.results_dir,
'fastproc_integration_ids')
try:
os.makedirs(self.autoproc_ids_dir)
except OSError: # it's there
pass
# we'll need the low res limit later on
lowres = data_in.low_resolution_limit
if lowres is not None:
self.low_resolution_limit = lowres.value
else:
self.low_resolution_limit = 50
res_override = data_in.res_override
if res_override is not None:
self.res_override = res_override.value
else:
# XXX: default to 0?
self.res_override = None
# check the number of images (must be > 8) and get the first
# image name to wait for. Also modify the XDS.INP file to
# reflect these values, if specified
conf = parse_xds_file(data_in.input_file.path.value)
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in conf:
xgeo = os.path.abspath(
os.path.join(self.root_dir, conf['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(xgeo))
del conf['X-GEO_CORR=']
else:
conf['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in conf:
ygeo = os.path.abspath(
os.path.join(self.root_dir, conf['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(ygeo))
del conf['Y-GEO_CORR=']
else:
conf['Y-GEO_CORR='] = ygeo
dump_xds_file(data_in.input_file.path.value, conf)
resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')
if resrange is not None:
if self.low_resolution_limit is not None:
resrange[0] = self.low_resolution_limit
if self.res_override is not None:
resrange[1] = self.res_override
conf['INCLUDE_RESOLUTION_RANGE='] = resrange
dump_xds_file(data_in.input_file.path.value, conf)
data_range = conf.get('DATA_RANGE=')
# we'll need that for the very last part ( file import )
self.data_range = data_range
if data_range is not None:
start_image = data_range[0]
end_image = data_range[1]
if end_image - start_image < 8:
self.ERROR('there are fewer than 8 images, aborting')
self.setFailure()
return
template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
self.DEBUG('template for images is {0}'.format(template))
# fix the path if it's not absolute
if not os.path.isabs(template):
self.DEBUG('file template {0} is not absolute'.format(template))
base_dir = os.path.abspath(
os.path.dirname(data_in.input_file.path.value))
template = os.path.normpath(os.path.join(self.root_dir, template))
conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template
self.DEBUG('file template fixed to {0}'.format(template))
self.DEBUG('dumping back the file to {0}'.format(
data_in.input_file.path.value))
dump_xds_file(data_in.input_file.path.value, conf)
first_image = _template_to_image(template, start_image)
self.wait_file = self.loadPlugin('EDPluginWaitFile')
waitfileinput = XSDataInputWaitFile()
waitfileinput.expectedFile = XSDataFile()
waitfileinput.expectedFile.path = XSDataString(first_image)
waitfileinput.expectedSize = XSDataInteger(0) # we do not care
timeout = XSDataTime()
global WAIT_FOR_FRAME_TIMEOUT
timeout.value = WAIT_FOR_FRAME_TIMEOUT
waitfileinput.timeOut = timeout
self.wait_file.dataInput = waitfileinput
self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")
self.store_autoproc_anom = self.loadPlugin(
'EDPluginISPyBStoreAutoProcv1_4')
self.store_autoproc_noanom = self.loadPlugin(
'EDPluginISPyBStoreAutoProcv1_4')
self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')
self.DEBUG('EDPluginControlAutoproc.preProcess finished')
