def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlXdsBest.preProcess")
self._xds = self.loadPlugin('EDPluginExecMinimalXds')
# save the root path (where the initial xds.inp is) for later use
self.root_dir = os.path.abspath(os.path.dirname(self.dataInput.input_file.value))
# let's begin by copying the input file to avoid clobbering it
shutil.copy(self.dataInput.input_file.value,
self.getWorkingDirectory())
self.real_input_file = os.path.join(self.getWorkingDirectory(),
os.path.basename(self.dataInput.input_file.value))
# update the keywords
self.parsed_config = parse_xds_file(self.real_input_file)
di = self.dataInput
unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
di.unit_cell_a.value,
di.unit_cell_b.value,
di.unit_cell_c.value,
di.unit_cell_alpha.value,
di.unit_cell_beta.value,
di.unit_cell_gamma.value)
sg = int(di.space_group.value)
self.parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
self.parsed_config['SPACE_GROUP_NUMBER='] = sg
# to avoid any problem let's also make the image template
# absolute as well. The underlying XDS template will take care
# of creating the links in its dir
imtemplate = self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
basedir = os.path.abspath(os.path.dirname(self.dataInput.input_file.value))
newpath = os.path.join(basedir, imtemplate)
self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in self.parsed_config:
xgeo = os.path.abspath(os.path.join(self.root_dir,
self.parsed_config['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
del self.parsed_config['X-GEO_CORR=']
else:
self.parsed_config['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in self.parsed_config:
ygeo = os.path.abspath(os.path.join(self.root_dir,
self.parsed_config['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
del self.parsed_config['Y-GEO_CORR=']
else:
self.parsed_config['Y-GEO_CORR='] = ygeo
dump_xds_file(self.real_input_file, self.parsed_config)
# create the input data model for the XDS plugin
input_dm = XSDataMinimalXdsIn()
input_dm.input_file = XSDataString(self.real_input_file)
self._xds.dataInput = input_dm
def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlXdsBest.process")
self._xds.executeSynchronous()
expected = os.path.join(self._xds.getWorkingDirectory(), 'XDS_ASCII.HKL')
if not os.path.exists(expected):
# Try to re-run with JOB = DEFPIX INTEGRATE CORRECT
self.parsed_config["JOB="] = "DEFPIX INTEGRATE CORRECT"
self._xds2 = self.loadPlugin('EDPluginExecMinimalXds')
self._xds2.setWorkingDirectory(self._xds.getWorkingDirectory())
dump_xds_file(self.real_input_file, self.parsed_config)
input_dm2 = XSDataMinimalXdsIn()
input_dm2.input_file = XSDataString(self.real_input_file)
self._xds2.dataInput = input_dm2
self._xds2.executeSynchronous()
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlXdsBest.process")
self._xds.executeSynchronous()
expected = os.path.join(self._xds.getWorkingDirectory(),
'XDS_ASCII.HKL')
if not os.path.exists(expected):
# Try to re-run with JOB = DEFPIX INTEGRATE CORRECT
self.parsed_config["JOB="] = "DEFPIX INTEGRATE CORRECT"
self._xds2 = self.loadPlugin('EDPluginExecMinimalXds')
self._xds2.setWorkingDirectory(self._xds.getWorkingDirectory())
dump_xds_file(self.real_input_file, self.parsed_config)
input_dm2 = XSDataMinimalXdsIn()
input_dm2.input_file = XSDataString(self.real_input_file)
self._xds2.dataInput = input_dm2
self._xds2.executeSynchronous()
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlAutoproc.preProcess")
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
self.xds_first = self.loadPlugin("EDPluginControlRunXds")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_anom = self.loadPlugin("EDPluginControlXscaleGenerate")
self.xscale_noanom = self.loadPlugin("EDPluginControlXscaleGenerate")
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlXdsBest.preProcess")
self._xds = self.loadPlugin('EDPluginExecMinimalXds')
# let's begin by copying the input file to avoid clobbering it
shutil.copy(self.dataInput.input_file.value,
self.getWorkingDirectory())
real_input_file = os.path.join(self.getWorkingDirectory(),
os.path.basename(self.dataInput.input_file.value))
# update the keywords
parsed_config = parse_xds_file(real_input_file)
di = self.dataInput
unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
di.unit_cell_a.value,
di.unit_cell_b.value,
di.unit_cell_c.value,
di.unit_cell_alpha.value,
di.unit_cell_beta.value,
di.unit_cell_gamma.value)
sg = int(di.space_group.value)
parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
parsed_config['SPACE_GROUP_NUMBER='] = sg
# to avoid any problem let's also make the image template
# absolute as well. The underlying XDS template will take care
# of creating the links in its dir
imtemplate = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
basedir = os.path.abspath(self.dataInput.input_file.value)
newpath = os.path.join(basedir, imtemplate)
parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
dump_xds_file(real_input_file, parsed_config)
# create the input data model for the XDS plugin
input_dm = XSDataMinimalXdsIn()
input_dm.input_file = XSDataString(real_input_file)
self._xds.dataInput = input_dm
def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXds.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# second run w/ JOB set to DEFPIX INTEGRATE CORRECT
self.second_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1
self.third_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'ALL'
params.maxprocs = 4
params.maxjobs = 1
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# final run with parallelism like 3 but JOB like 2
self.final_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
params.maxprocs = 4
params.maxjobs = 1
self.final_run.dataInput = params
self.final_run.executeSynchronous()
EDVerbose.DEBUG('final run completed')
if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.final_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlAutoproc.preProcess")
self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')
path = os.path.abspath(self.dataInput.previous_run_dir.value)
# create the data inputs now we know the files are here
input_anom = XSDataMinimalXdsIn()
input_anom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
input_anom.friedels_law = XSDataBoolean(True)
input_anom.job = XSDataString('CORRECT')
input_anom.resolution = self.dataInput.resolution
input_anom.resolution_range = [
XSDataFloat(60), self.dataInput.resolution
]
self.xds_anom.dataInput = input_anom
input_noanom = XSDataMinimalXdsIn()
input_noanom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
input_noanom.fridels_law = XSDataBoolean(False)
input_noanom.job = XSDataString('CORRECT')
input_noanom.resolution_range = [
XSDataFloat(60), self.dataInput.resolution
]
self.xds_noanom.dataInput = input_noanom
xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())
# let's make some links!
for f in self._to_link:
os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
# now this is the horrible part, we need to make symlinks to
# the images also. for now we rely on the fact that the links
# in the previous run are most likely the links to the
# images. So we will make the same links and rely on the fact
# that in the input file the path to the images is already
# relative to the CWD
path = os.path.abspath(self.dataInput.previous_run_dir.value)
for f in os.listdir(path):
fullpath = os.path.join(path, f)
if os.path.islink(fullpath):
# symlink the symlink...
os.symlink(fullpath, os.path.join(xds_anom_dir, f))
os.symlink(fullpath, os.path.join(xds_noanom_dir, f))
def process(self, _edObject = None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXdsFastProc.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.second_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG('copying previously generated files to the new plugin dir')
copy_xds_files(self.first_run.getWorkingDirectory(),
self.second_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
# increase all the spot ranges end by 20, constrained to
# the data range upper limit
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 20
if range_end > self.end_image_no:
self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.third_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG('copying previously generated files to the new plugin dir')
copy_xds_files(self.second_run.getWorkingDirectory(),
self.third_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 40
if range_end > self.end_image_no:
self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXdsFastProc.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.second_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG(
'copying previously generated files to the new plugin dir')
copy_xds_files(self.first_run.getWorkingDirectory(),
self.second_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
# increase all the spot ranges end by 20, constrained to
# the data range upper limit
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 20
if range_end > self.end_image_no:
self.DEBUG(
'End of range {0} would be past the last image {1}'.
format(range_end, self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
self.third_run = self.loadPlugin(self.controlled_plugin_name)
self.DEBUG('retrying with increased SPOT_RANGE')
self.DEBUG(
'copying previously generated files to the new plugin dir')
copy_xds_files(self.second_run.getWorkingDirectory(),
self.third_run.getWorkingDirectory())
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
params.spacegroup = self.dataInput.spacegroup
params.unit_cell = self.dataInput.unit_cell
spot_range = list()
for srange in self.spot_range:
range_begin = srange[0]
range_end = srange[1] + 40
if range_end > self.end_image_no:
self.DEBUG(
'End of range {0} would be past the last image {1}'.
format(range_end, self.end_image_no))
range_end = self.end_image_no
self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
srange, range_begin, range_end))
r = XSDataRange(begin=range_begin, end=range_end)
spot_range.append(r)
params.spot_range = spot_range
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def process(self, _edObject=None):
EDPluginControl.process(self)
self.DEBUG("EDPluginControlRunXds.process")
# First run is vanilla without any modification
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
self.first_run.dataInput = params
self.first_run.executeSynchronous()
EDVerbose.DEBUG('first run completed...')
if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.first_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# second run w/ JOB set to DEFPIX INTEGRATE CORRECT
self.second_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
self.second_run.dataInput = params
self.second_run.executeSynchronous()
EDVerbose.DEBUG('second run completed')
if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.second_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1
self.third_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'ALL'
params.maxprocs = 4
params.maxjobs = 1
self.third_run.dataInput = params
self.third_run.executeSynchronous()
EDVerbose.DEBUG('third run completed')
if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.third_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# final run with parallelism like 3 but JOB like 2
self.final_run = self.loadPlugin(self.controlled_plugin_name)
params = XSDataMinimalXdsIn()
params.input_file = self.dataInput.input_file
params.jobs = 'DEFPIX INTEGRATE CORRECT'
params.maxprocs = 4
params.maxjobs = 1
self.final_run.dataInput = params
self.final_run.executeSynchronous()
EDVerbose.DEBUG('final run completed')
if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded:
EDVerbose.DEBUG('... and it worked')
self.successful_run = self.final_run
else:
EDVerbose.DEBUG('... and it failed')
if not self.successful_run:
# all runs failed so bail out ...
self.setFailure()
else:
# use the xds parser plugin to parse the xds output file...
parser = self.loadPlugin("EDPluginParseXdsOutput")
wd = self.successful_run.getWorkingDirectory()
parser_input = XSDataXdsOutputFile()
correct_lp_path = XSDataFile()
correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
parser_input.correct_lp = correct_lp_path
gxparm_path = os.path.join(wd, 'GXPARM.XDS')
if os.path.isfile(gxparm_path):
gxparm = XSDataFile()
gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
parser_input.gxparm = gxparm
parser.dataInput = parser_input
parser.executeSynchronous()
if parser.isFailure():
# that should not happen
self.setFailure()
return
self.dataOutput = parser.dataOutput
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlAutoproc.preProcess")
self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')
path = os.path.abspath(self.dataInput.previous_run_dir.value)
# The MinimalXds plugin takes care of creating determining the
# real images directory and creating a symlink to it so we
# only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
# to not be relative anymore. We'll copy it to our own dir
# beforehand to avoid clobbering it
xdsinp = os.path.join(path, 'XDS.INP')
new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
copyfile(xdsinp, new_xdsinp)
parsed_config = parse_xds_file(new_xdsinp)
file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(os.path.join(path, file_template))
dump_xds_file(new_xdsinp, parsed_config)
# create the data inputs now we know the files are here
input_anom = XSDataMinimalXdsIn()
input_anom.input_file = XSDataString(new_xdsinp)
input_anom.friedels_law = XSDataBoolean(False)
input_anom.job = XSDataString('CORRECT')
input_anom.resolution = self.dataInput.resolution
input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
self.xds_anom.dataInput = input_anom
input_noanom = XSDataMinimalXdsIn()
input_noanom.input_file = XSDataString(new_xdsinp)
input_noanom.friedels_law = XSDataBoolean(True)
input_noanom.job = XSDataString('CORRECT')
input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
self.xds_noanom.dataInput = input_noanom
xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())
# let's make some links!
for f in self._to_link:
os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG('EDPluginControlAutoproc.preProcess starting')
self.DEBUG('failure state is currently {0}'.format(self.isFailure()))
# for info to send to the autoproc stats server
self.custom_stats = dict(creation_time=time.time(),
processing_type='edna fastproc',
datacollect_id=self.dataInput.data_collection_id.value,
comments='running on {0}'.format(socket.gethostname()))
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
xds_in.spacegroup = data_in.spacegroup
xds_in.unit_cell = data_in.unit_cell
self.log_file_path = os.path.join(self.root_dir, 'stats.json')
self.DEBUG('will log timing information to {0}'.format(self.log_file_path))
self.stats = dict()
# Get the image prefix from the directory name
# XXX: This is horrible
try:
self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1])
except Exception:
self.image_prefix = ''
self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing')
try:
os.makedirs(self.results_dir)
except OSError: # it most likely exists
pass
# Copy the vanilla XDS input file to the results dir
infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP')
shutil.copy(self.dataInput.input_file.path.value,
infile_dest)
# Ensure the autoproc ids directory is there
self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids')
try:
os.makedirs(self.autoproc_ids_dir)
except OSError: # it's there
pass
# we'll need the low res limit later on
lowres = data_in.low_resolution_limit
if lowres is not None:
self.low_resolution_limit = lowres.value
else:
self.low_resolution_limit = 50
res_override = data_in.res_override
if res_override is not None:
self.res_override = res_override.value
else:
# XXX: default to 0?
self.res_override = None
# check the number of images (must be > 8) and get the first
# image name to wait for. Also modify the XDS.INP file to
# reflect these values, if specified
conf = parse_xds_file(data_in.input_file.path.value)
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in conf:
xgeo = os.path.abspath(os.path.join(self.root_dir,
conf['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
del conf['X-GEO_CORR=']
else:
conf['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in conf:
ygeo = os.path.abspath(os.path.join(self.root_dir,
conf['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
del conf['Y-GEO_CORR=']
else:
conf['Y-GEO_CORR='] = ygeo
dump_xds_file(data_in.input_file.path.value, conf)
resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')
if resrange is not None:
if self.low_resolution_limit is not None:
resrange[0] = self.low_resolution_limit
if self.res_override is not None:
resrange[1] = self.res_override
conf['INCLUDE_RESOLUTION_RANGE='] = resrange
dump_xds_file(data_in.input_file.path.value, conf)
data_range = conf.get('DATA_RANGE=')
# we'll need that for the very last part ( file import )
self.data_range = data_range
if data_range is not None:
start_image = data_range[0]
end_image = data_range[1]
if end_image - start_image < 8:
self.ERROR('there are fewer than 8 images, aborting')
self.setFailure()
return
template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
self.DEBUG('template for images is {0}'.format(template))
# fix the path if it's not absolute
if not os.path.isabs(template):
self.DEBUG('file template {0} is not absolute'.format(template))
base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value))
template = os.path.normpath(os.path.join(self.root_dir, template))
conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template
self.DEBUG('file template fixed to {0}'.format(template))
self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value))
dump_xds_file(data_in.input_file.path.value, conf)
first_image = _template_to_image(template, start_image)
self.wait_file = self.loadPlugin('EDPluginWaitFile')
waitfileinput = XSDataInputWaitFile()
waitfileinput.expectedFile = XSDataFile()
waitfileinput.expectedFile.path = XSDataString(first_image)
waitfileinput.expectedSize = XSDataInteger(0) # we do not care
timeout = XSDataTime()
global WAIT_FOR_FRAME_TIMEOUT
timeout.value = WAIT_FOR_FRAME_TIMEOUT
waitfileinput.timeOut = timeout
self.wait_file.dataInput = waitfileinput
self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")
self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')
self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlAutoproc.preProcess")
self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')
path = os.path.abspath(self.dataInput.previous_run_dir.value)
# create the data inputs now we know the files are here
input_anom = XSDataMinimalXdsIn()
input_anom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
input_anom.friedels_law = XSDataBoolean(True)
input_anom.job = XSDataString('CORRECT')
input_anom.resolution = self.dataInput.resolution
input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
self.xds_anom.dataInput = input_anom
input_noanom = XSDataMinimalXdsIn()
input_noanom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
input_noanom.fridels_law = XSDataBoolean(False)
input_noanom.job = XSDataString('CORRECT')
input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
self.xds_noanom.dataInput = input_noanom
xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())
# let's make some links!
for f in self._to_link:
os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
# now this is the horrible part, we need to make symlinks to
# the images also. for now we rely on the fact that the links
# in the previous run are most likely the links to the
# images. So we will make the same links and rely on the fact
# that in the input file the path to the images is already
# relative to the CWD
path = os.path.abspath(self.dataInput.previous_run_dir.value)
for f in os.listdir(path):
fullpath = os.path.join(path, f)
if os.path.islink(fullpath):
# symlink the symlink...
os.symlink(fullpath, os.path.join(xds_anom_dir, f))
os.symlink(fullpath, os.path.join(xds_noanom_dir, f))
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlXdsBest.preProcess")
self._xds = self.loadPlugin('EDPluginExecMinimalXds')
# save the root path (where the initial xds.inp is) for later use
self.root_dir = os.path.abspath(
os.path.dirname(self.dataInput.input_file.value))
# let's begin by copying the input file to avoid clobbering it
shutil.copy(self.dataInput.input_file.value,
self.getWorkingDirectory())
self.real_input_file = os.path.join(
self.getWorkingDirectory(),
os.path.basename(self.dataInput.input_file.value))
# update the keywords
self.parsed_config = parse_xds_file(self.real_input_file)
di = self.dataInput
unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
di.unit_cell_a.value, di.unit_cell_b.value, di.unit_cell_c.value,
di.unit_cell_alpha.value, di.unit_cell_beta.value,
di.unit_cell_gamma.value)
sg = int(di.space_group.value)
self.parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
self.parsed_config['SPACE_GROUP_NUMBER='] = sg
# to avoid any problem let's also make the image template
# absolute as well. The underlying XDS template will take care
# of creating the links in its dir
imtemplate = self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
basedir = os.path.abspath(
os.path.dirname(self.dataInput.input_file.value))
newpath = os.path.join(basedir, imtemplate)
self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in self.parsed_config:
xgeo = os.path.abspath(
os.path.join(self.root_dir,
self.parsed_config['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(xgeo))
del self.parsed_config['X-GEO_CORR=']
else:
self.parsed_config['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in self.parsed_config:
ygeo = os.path.abspath(
os.path.join(self.root_dir,
self.parsed_config['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(ygeo))
del self.parsed_config['Y-GEO_CORR=']
else:
self.parsed_config['Y-GEO_CORR='] = ygeo
dump_xds_file(self.real_input_file, self.parsed_config)
# create the input data model for the XDS plugin
input_dm = XSDataMinimalXdsIn()
input_dm.input_file = XSDataString(self.real_input_file)
self._xds.dataInput = input_dm
def preProcess(self, _edObject=None):
EDPluginControl.preProcess(self)
self.DEBUG('EDPluginControlAutoproc.preProcess starting')
self.DEBUG('failure state is currently {0}'.format(self.isFailure()))
# for info to send to the autoproc stats server
self.custom_stats = dict(
creation_time=time.time(),
processing_type='edna fastproc',
datacollect_id=self.dataInput.data_collection_id.value,
comments='running on {0}'.format(socket.gethostname()))
data_in = self.dataInput
xds_in = XSDataMinimalXdsIn()
xds_in.input_file = data_in.input_file.path
xds_in.spacegroup = data_in.spacegroup
xds_in.unit_cell = data_in.unit_cell
self.log_file_path = os.path.join(self.root_dir, 'stats.json')
self.DEBUG('will log timing information to {0}'.format(
self.log_file_path))
self.stats = dict()
# Get the image prefix from the directory name
# XXX: This is horrible
try:
self.image_prefix = '_'.join(
os.path.basename(self.root_dir).split('_')[1:-1])
except Exception:
self.image_prefix = ''
self.results_dir = os.path.join(self.root_dir, 'results',
'fast_processing')
try:
os.makedirs(self.results_dir)
except OSError: # it most likely exists
pass
# Copy the vanilla XDS input file to the results dir
infile_dest = os.path.join(self.results_dir,
self.image_prefix + '_input_XDS.INP')
shutil.copy(self.dataInput.input_file.path.value, infile_dest)
# Ensure the autoproc ids directory is there
self.autoproc_ids_dir = os.path.join(self.results_dir,
'fastproc_integration_ids')
try:
os.makedirs(self.autoproc_ids_dir)
except OSError: # it's there
pass
# we'll need the low res limit later on
lowres = data_in.low_resolution_limit
if lowres is not None:
self.low_resolution_limit = lowres.value
else:
self.low_resolution_limit = 50
res_override = data_in.res_override
if res_override is not None:
self.res_override = res_override.value
else:
# XXX: default to 0?
self.res_override = None
# check the number of images (must be > 8) and get the first
# image name to wait for. Also modify the XDS.INP file to
# reflect these values, if specified
conf = parse_xds_file(data_in.input_file.path.value)
# Make the [XY]-GEO_CORR paths absolute
if 'X-GEO_CORR=' in conf:
xgeo = os.path.abspath(
os.path.join(self.root_dir, conf['X-GEO_CORR='][0]))
if not os.path.exists(xgeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(xgeo))
del conf['X-GEO_CORR=']
else:
conf['X-GEO_CORR='] = xgeo
if 'Y-GEO_CORR=' in conf:
ygeo = os.path.abspath(
os.path.join(self.root_dir, conf['Y-GEO_CORR='][0]))
if not os.path.exists(ygeo):
self.DEBUG(
'geometry file {0} does not exist, removing'.format(ygeo))
del conf['Y-GEO_CORR=']
else:
conf['Y-GEO_CORR='] = ygeo
dump_xds_file(data_in.input_file.path.value, conf)
resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')
if resrange is not None:
if self.low_resolution_limit is not None:
resrange[0] = self.low_resolution_limit
if self.res_override is not None:
resrange[1] = self.res_override
conf['INCLUDE_RESOLUTION_RANGE='] = resrange
dump_xds_file(data_in.input_file.path.value, conf)
data_range = conf.get('DATA_RANGE=')
# we'll need that for the very last part ( file import )
self.data_range = data_range
if data_range is not None:
start_image = data_range[0]
end_image = data_range[1]
if end_image - start_image < 8:
self.ERROR('there are fewer than 8 images, aborting')
self.setFailure()
return
template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
self.DEBUG('template for images is {0}'.format(template))
# fix the path if it's not absolute
if not os.path.isabs(template):
self.DEBUG('file template {0} is not absolute'.format(template))
base_dir = os.path.abspath(
os.path.dirname(data_in.input_file.path.value))
template = os.path.normpath(os.path.join(self.root_dir, template))
conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template
self.DEBUG('file template fixed to {0}'.format(template))
self.DEBUG('dumping back the file to {0}'.format(
data_in.input_file.path.value))
dump_xds_file(data_in.input_file.path.value, conf)
first_image = _template_to_image(template, start_image)
self.wait_file = self.loadPlugin('EDPluginWaitFile')
waitfileinput = XSDataInputWaitFile()
waitfileinput.expectedFile = XSDataFile()
waitfileinput.expectedFile.path = XSDataString(first_image)
waitfileinput.expectedSize = XSDataInteger(0) # we do not care
timeout = XSDataTime()
global WAIT_FOR_FRAME_TIMEOUT
timeout.value = WAIT_FOR_FRAME_TIMEOUT
waitfileinput.timeOut = timeout
self.wait_file.dataInput = waitfileinput
self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
self.xds_first.dataInput = xds_in
self.generate = self.loadPlugin("EDPluginXDSGenerate")
self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")
self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")
self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")
self.store_autoproc_anom = self.loadPlugin(
'EDPluginISPyBStoreAutoProcv1_4')
self.store_autoproc_noanom = self.loadPlugin(
'EDPluginISPyBStoreAutoProcv1_4')
self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')
self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def preProcess(self, _edObject = None):
EDPluginControl.preProcess(self)
self.DEBUG("EDPluginControlAutoproc.preProcess")
self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')
path = os.path.abspath(self.dataInput.previous_run_dir.value)
# The MinimalXds plugin takes care of creating determining the
# real images directory and creating a symlink to it so we
# only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
# to not be relative anymore. We'll copy it to our own dir
# beforehand to avoid clobbering it
xdsinp = os.path.join(path, 'XDS.INP')
new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
copyfile(xdsinp, new_xdsinp)
parsed_config = parse_xds_file(new_xdsinp)
file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(os.path.join(path, file_template))
dump_xds_file(new_xdsinp, parsed_config)
# create the data inputs now we know the files are here
input_anom = XSDataMinimalXdsIn()
input_anom.input_file = XSDataString(new_xdsinp)
input_anom.friedels_law = XSDataBoolean(True)
input_anom.job = XSDataString('CORRECT')
input_anom.resolution = self.dataInput.resolution
input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
self.xds_anom.dataInput = input_anom
input_noanom = XSDataMinimalXdsIn()
input_noanom.input_file = XSDataString(new_xdsinp)
input_noanom.friedels_law = XSDataBoolean(False)
input_noanom.job = XSDataString('CORRECT')
input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
self.xds_noanom.dataInput = input_noanom
xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())
# let's make some links!
for f in self._to_link:
os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))