def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginXDSGeneratev1_0.preProcess")

        if self.dataInput.doAnomAndNonanom is not None:
            if self.dataInput.doAnomAndNonanom.value:
                self.doAnomAndNonanom = True
            else:
                self.doAnomAndNonanom = False

        self.xds_anom = self.loadPlugin('EDPluginExecMinimalXdsv1_0')
        if self.doAnomAndNonanom:
            self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXdsv1_0')

        path = os.path.abspath(self.dataInput.previous_run_dir.value)

        # The MinimalXds plugin takes care of creating determining the
        # real images directory and creating a symlink to it so we
        # only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
        # to not be relative anymore. We'll copy it to our own dir
        # beforehand to avoid clobbering it
        xdsinp = os.path.join(path, 'XDS.INP')
        new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
        copyfile(xdsinp, new_xdsinp)


        parsed_config = parse_xds_file(new_xdsinp)
        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(os.path.join(path, file_template))
        dump_xds_file(new_xdsinp, parsed_config)

        # create the data inputs now we know the files are here
        input_anom = XSDataMinimalXdsIn()
        input_anom.input_file = XSDataString(new_xdsinp)
        input_anom.friedels_law = XSDataBoolean(False)
        input_anom.job = XSDataString('CORRECT')
        input_anom.resolution = self.dataInput.resolution
        input_anom.resolution_range = [XSDataDouble(60), self.dataInput.resolution]
        input_anom.spacegroup = self.dataInput.spacegroup
        input_anom.unit_cell = self.dataInput.unit_cell
        self.xds_anom.dataInput = input_anom
        xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())

        if self.doAnomAndNonanom:
            input_noanom = XSDataMinimalXdsIn()
            input_noanom.input_file = XSDataString(new_xdsinp)
            input_noanom.friedels_law = XSDataBoolean(True)
            input_noanom.job = XSDataString('CORRECT')
            input_noanom.resolution_range = [XSDataDouble(60), self.dataInput.resolution]
            input_noanom.spacegroup = self.dataInput.spacegroup
            input_noanom.unit_cell = self.dataInput.unit_cell
            self.xds_noanom.dataInput = input_noanom
            xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())


        # let's make some links!
        for f in self._to_link:
            os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
            if self.doAnomAndNonanom:
                os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        input_file = self.dataInput.input_file
        if input_file is None:
            EDVerbose.ERROR('no xds input file given, giving up')
            self.setFailure()
            return

        input_file = input_file.value
        if not os.path.exists(input_file):
            EDVerbose.ERROR('the specified input file does not seem to exist,'
                            ' giving up')
            self.setFailure()
            return

        # we need to "fix" the input file so the SPOT_RANGE gets set
        # to a sensible value
        cfg = parse_xds_file(input_file)
        spot_range = _spot_range(cfg)
        if len(spot_range) != 0:
            cfg['SPOT_RANGE='] = spot_range

        #SECONDS is also set to 20
        cfg['SECONDS='] = 20
        dump_xds_file(input_file, cfg)
Beispiel #3
0
    def checkParameters(self):
        """
        Checks the mandatory parameters.
        """
        input_file = self.dataInput.input_file
        if input_file is None:
            EDVerbose.ERROR('no xds input file given, giving up')
            self.setFailure()
            return

        input_file = input_file.value
        if not os.path.exists(input_file):
            EDVerbose.ERROR('the specified input file does not seem to exist,'
                            ' giving up')
            self.setFailure()
            return

        # we need to "fix" the input file so the SPOT_RANGE gets set
        # to a sensible value
        cfg = parse_xds_file(input_file)
        spot_range = _spot_range(cfg)
        if len(spot_range) != 0:
            cfg['SPOT_RANGE='] = spot_range

        #SECONDS is also set to 20
        cfg['SECONDS='] = 20
        dump_xds_file(input_file, cfg)
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlXdsBest.preProcess")
        self._xds = self.loadPlugin('EDPluginExecMinimalXdsv1_0')

        # save the root path (where the initial xds.inp is) for later use
        self.root_dir = os.path.abspath(os.path.dirname(self.dataInput.input_file.value))

        # let's begin by copying the input file to avoid clobbering it
        shutil.copy(self.dataInput.input_file.value,
                    self.getWorkingDirectory())
        self.real_input_file = os.path.join(self.getWorkingDirectory(),
                                       os.path.basename(self.dataInput.input_file.value))

        # update the keywords
        self.parsed_config = parse_xds_file(self.real_input_file)
        di = self.dataInput
        unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
            di.unit_cell_a.value,
            di.unit_cell_b.value,
            di.unit_cell_c.value,
            di.unit_cell_alpha.value,
            di.unit_cell_beta.value,
            di.unit_cell_gamma.value)
        sg = int(di.space_group.value)
        self.parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
        self.parsed_config['SPACE_GROUP_NUMBER='] = sg

        # to avoid any problem let's also make the image template
        # absolute as well. The underlying XDS template will take care
        # of creating the links in its dir
        imtemplate = self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        basedir = os.path.abspath(os.path.dirname(self.dataInput.input_file.value))
        newpath = os.path.join(basedir, imtemplate)
        self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in self.parsed_config:
            xgeo = os.path.abspath(os.path.join(self.root_dir,
                                                self.parsed_config['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
                del self.parsed_config['X-GEO_CORR=']
            else:
                self.parsed_config['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in self.parsed_config:
            ygeo = os.path.abspath(os.path.join(self.root_dir,
                                                self.parsed_config['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
                del self.parsed_config['Y-GEO_CORR=']
            else:
                self.parsed_config['Y-GEO_CORR='] = ygeo
        dump_xds_file(self.real_input_file, self.parsed_config)

        # create the input data model for the XDS plugin
        input_dm = XSDataMinimalXdsIn()
        input_dm.input_file = XSDataString(self.real_input_file)
        self._xds.dataInput = input_dm
Beispiel #5
0
    def testParse_xds_file(self):
        xdsInpPath = os.path.join(self.getPluginTestsDataHome(), "XDS_xdscfgparser.INP")
        parserDict = xdscfgparser.parse_xds_file(xdsInpPath)
#        import pprint
#        pprint.pprint(parserDict)
        EDAssert.equal([1, 4], parserDict["BACKGROUND_RANGE="], "BACKGROUND_RANGE")
        EDAssert.equal(['/data/id29/inhouse/opid291/x_geo_corr.cbf'], parserDict["X-GEO_CORR="], "X-GEO_CORR")
        EDAssert.strAlmostEqual(305.04, parserDict['DETECTOR_DISTANCE='], 'DETECTOR_DISTANCE')
Beispiel #6
0
    def preProcess(self, _edObject=None):
        EDPluginExecProcessScript.preProcess(self)
        self.DEBUG("EDPluginMinimalXDS.preProcess")
        xds_input = os.path.abspath(self.dataInput.input_file.value)
        shutil.copy(xds_input, self.getWorkingDirectory())

        # our new xds file
        xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')

        parsed_config = parse_xds_file(xds_file)

        # try to make some symlinks to work around the path length
        # limitation of xds
        # TODO: don't even try if not on Unix

        # XXX: why did i make this config item a list instead of a
        # regular string?
        file_template = os.path.abspath(
            parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0])

        directory = os.path.dirname(file_template)
        filename = os.path.basename(file_template)

        matches = fnmatch.filter(os.listdir(directory), filename)
        our_dir = self.getWorkingDirectory()
        for f in matches:
            os.symlink(os.path.join(directory, f), os.path.join(our_dir, f))
        # patch the template in the config by stripping the whole prefix
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = filename

        # perhaps modify some params
        job = self.dataInput.job
        maxproc = self.dataInput.maxproc
        maxjobs = self.dataInput.maxjobs
        resolution_range = self.dataInput.resolution_range
        friedels_law = self.dataInput.friedels_law

        if job is not None:
            parsed_config["JOB="] = job.value
        if maxproc is not None:
            parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
        if maxjobs is not None:
            parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
        if resolution_range is not None and len(resolution_range) != 0:
            parsed_config["INCLUDE_RESOLUTION_RANGE="] = [
                x.value for x in resolution_range
            ]
        if friedels_law is not None:
            if friedels_law:
                parsed_config["FRIEDEL'S_LAW="] = "TRUE"
            else:
                parsed_config["FRIEDEL'S_LAW="] = "FALSE"
        dump_xds_file(xds_file, parsed_config)
    def preProcess(self, _edObject = None):
        EDPluginExecProcessScript.preProcess(self)
        self.DEBUG("EDPluginMinimalXDS.preProcess")
        xds_input = os.path.abspath(self.dataInput.input_file.value)
        shutil.copy(xds_input, self.getWorkingDirectory())

        # our new xds file
        xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')

        parsed_config = parse_xds_file(xds_file)

        # try to make some symlinks to work around the path length
        # limitation of xds
        # TODO: don't even try if not on Unix

        # XXX: why did i make this config item a list instead of a
        # regular string?
        file_template = os.path.abspath(parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0])

        directory = os.path.dirname(file_template)
        filename = os.path.basename(file_template)

        matches = fnmatch.filter(os.listdir(directory), filename)
        our_dir = self.getWorkingDirectory()
        for f in matches:
            os.symlink(os.path.join(directory, f),
                       os.path.join(our_dir, f))
        # patch the template in the config by stripping the whole prefix
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = filename

        # perhaps modify some params
        job = self.dataInput.job
        maxproc = self.dataInput.maxproc
        maxjobs = self.dataInput.maxjobs
        resolution_range = self.dataInput.resolution_range
        friedels_law = self.dataInput.friedels_law

        if job is not None:
            parsed_config["JOB="] = job.value
        if maxproc is not None:
            parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
        if maxjobs is not None:
            parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
        if resolution_range is not None and len(resolution_range) != 0:
            parsed_config["INCLUDE_RESOLUTION_RANGE="] = [x.value for x in resolution_range]
        if friedels_law is not None:
            if friedels_law:
                parsed_config["FRIEDEL'S_LAW="] = "TRUE"
            else:
                parsed_config["FRIEDEL'S_LAW="] = "FALSE"
        dump_xds_file(xds_file, parsed_config)
Beispiel #8
0
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAutoproc.preProcess")

        self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
        self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')

        path = os.path.abspath(self.dataInput.previous_run_dir.value)

        # The MinimalXds plugin takes care of creating determining the
        # real images directory and creating a symlink to it so we
        # only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
        # to not be relative anymore. We'll copy it to our own dir
        # beforehand to avoid clobbering it
        xdsinp = os.path.join(path, 'XDS.INP')
        new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
        copyfile(xdsinp, new_xdsinp)


        parsed_config = parse_xds_file(new_xdsinp)
        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(os.path.join(path, file_template))
        dump_xds_file(new_xdsinp, parsed_config)

        # create the data inputs now we know the files are here
        input_anom = XSDataMinimalXdsIn()
        input_anom.input_file = XSDataString(new_xdsinp)
        input_anom.friedels_law = XSDataBoolean(True)
        input_anom.job = XSDataString('CORRECT')
        input_anom.resolution = self.dataInput.resolution
        input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_anom.dataInput = input_anom

        input_noanom = XSDataMinimalXdsIn()
        input_noanom.input_file = XSDataString(new_xdsinp)
        input_noanom.friedels_law = XSDataBoolean(False)
        input_noanom.job = XSDataString('CORRECT')
        input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_noanom.dataInput = input_noanom

        xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
        xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())


        # let's make some links!
        for f in self._to_link:
            os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
            os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlRunXdsFastProcv1_0.preProcess")
        # Load the execution plugin
        self.first_run = self.loadPlugin(self.controlled_plugin_name)

        cfg = parse_xds_file(self.dataInput.input_file.value)
        spot_range = cfg.get('SPOT_RANGE=')
        if spot_range is None:
            if EDUtilsPath.isEMBL():
                # GB: add a default here - it is nothing worthe then the
                #    one in autogenerated XDS.INP
                pass
            else:
                strErrorMessage = "No SPOT_RANGE parameter"
                self.addErrorMessage(strErrorMessage)
                self.ERROR(strErrorMessage)
                self.setFailure()
        else:
            self.spot_range = spot_range

        # we will use this value to constrain the upper bound of
        # spot_range so it does not get past the last image number, so
        # we use a default value that cannot be a constraint in case
        # we cannot find it in the xds input file
        self.end_image_no = sys.maxsize

        data_range = cfg.get('DATA_RANGE=')
        if data_range is not None:
            self.start_image_no = data_range[0]
            self.end_image_no = data_range[1]

        # GB: repair the above....  but why spot_range should be a list of lists?
        if spot_range is None:
            self.spot_range = [[
                max(1, data_range[0]),
                min(data_range[0] + 99, data_range[1])
            ]]
        else:
            # Check that spot ranges is within data range
            new_spot_range = []
            for spot_range in self.spot_range:
                if spot_range[0] >= self.start_image_no and spot_range[
                        1] <= self.end_image_no:
                    new_spot_range.append(spot_range)
            self.spot_range = new_spot_range
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlRunXdsFastProcv1_0.preProcess")
        # Load the execution plugin
        self.first_run = self.loadPlugin(self.controlled_plugin_name)

        cfg = parse_xds_file(self.dataInput.input_file.value)
        spot_range = cfg.get('SPOT_RANGE=')
        if spot_range is None:
            if EDUtilsPath.isEMBL():
               # GB: add a default here - it is nothing worthe then the
               #    one in autogenerated XDS.INP
               pass
            else:
               strErrorMessage = "No SPOT_RANGE parameter"
               self.addErrorMessage(strErrorMessage)
               self.ERROR(strErrorMessage)
               self.setFailure()
        else:
            self.spot_range = spot_range

        # we will use this value to constrain the upper bound of
        # spot_range so it does not get past the last image number, so
        # we use a default value that cannot be a constraint in case
        # we cannot find it in the xds input file
        self.end_image_no = sys.maxint

        data_range = cfg.get('DATA_RANGE=')
        if data_range is not None:
            self.start_image_no = data_range[0]
            self.end_image_no = data_range[1]

        # GB: repair the above....  but why spot_range should be a list of lists?
        if spot_range is None:
           self.spot_range = [[max(1, data_range[0]),
                               min(data_range[0] + 99, data_range[1])]]
        else:
            # Check that spot ranges is within data range
            new_spot_range = []
            for spot_range in self.spot_range:
                if spot_range[0] >= self.start_image_no and spot_range[1] <= self.end_image_no:
                    new_spot_range.append(spot_range)
            self.spot_range = new_spot_range
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlXdsBest.preProcess")
        self._xds = self.loadPlugin('EDPluginExecMinimalXds')

        # let's begin by copying the input file to avoid clobbering it
        shutil.copy(self.dataInput.input_file.value,
                    self.getWorkingDirectory())
        real_input_file = os.path.join(self.getWorkingDirectory(),
                                       os.path.basename(self.dataInput.input_file.value))
        # update the keywords
        parsed_config = parse_xds_file(real_input_file)
        di = self.dataInput
        unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
            di.unit_cell_a.value,
            di.unit_cell_b.value,
            di.unit_cell_c.value,
            di.unit_cell_alpha.value,
            di.unit_cell_beta.value,
            di.unit_cell_gamma.value)
        sg = int(di.space_group.value)
        parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
        parsed_config['SPACE_GROUP_NUMBER='] = sg

        # to avoid any problem let's also make the image template
        # absolute as well. The underlying XDS template will take care
        # of creating the links in its dir
        imtemplate = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        basedir = os.path.abspath(self.dataInput.input_file.value)
        newpath = os.path.join(basedir, imtemplate)
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath

        dump_xds_file(real_input_file, parsed_config)

        # create the input data model for the XDS plugin
        input_dm = XSDataMinimalXdsIn()
        input_dm.input_file = XSDataString(real_input_file)
        self._xds.dataInput = input_dm
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlRunXdsFastProc.preProcess")
        # Load the execution plugin
        self.first_run = self.loadPlugin(self.controlled_plugin_name)

        cfg = parse_xds_file(self.dataInput.input_file.value)
        spot_range = cfg.get('SPOT_RANGE=')
        if spot_range is None:
            self.ERROR('no SPOT_RANGE parameter')
            self.setFailure()
        else:
            self.spot_range = spot_range

        # we will use this value to constrain the upper bound of
        # spot_range so it does not get past the last image number, so
        # we use a default value that cannot be a constraint in case
        # we cannot find it in the xds input file
        self.end_image_no = sys.maxint

        data_range = cfg.get('DATA_RANGE=')
        if data_range is not None:
            self.end_image_no = data_range[1]
Beispiel #13
0
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlRunXdsFastProc.preProcess")
        # Load the execution plugin
        self.first_run = self.loadPlugin(self.controlled_plugin_name)

        cfg = parse_xds_file(self.dataInput.input_file.value)
        spot_range = cfg.get('SPOT_RANGE=')
        if spot_range is None:
            self.ERROR('no SPOT_RANGE parameter')
            self.setFailure()
        else:
            self.spot_range = spot_range

        # we will use this value to constrain the upper bound of
        # spot_range so it does not get past the last image number, so
        # we use a default value that cannot be a constraint in case
        # we cannot find it in the xds input file
        self.end_image_no = sys.maxint

        data_range = cfg.get('DATA_RANGE=')
        if data_range is not None:
            self.end_image_no = data_range[1]
Beispiel #14
0
xds_appeared = False
wait_xds_start = time.time()
logging.debug('Waiting for XDS.INP file')
while not xds_appeared and time.time() - wait_xds_start < WAIT_XDS_TIMEOUT:
    if os.path.exists(input_file):
        xds_appeared = True
        logging.debug('XDS.INP file is there')
    else:
        time.sleep(WAIT_XDS_RESOLUTION)
if not xds_appeared:
    logging.error(
        'XDS.INP file ({0}) failed to appear after {1} seconds'.format(
            input_file, WAIT_XDS_TIMEOUT))
    sys.exit(1)

cfg = xdscfgparser.parse_xds_file(input_file)
last_image = file_template = None
try:
    last_image = cfg['DATA_RANGE='][1]
    file_template = cfg['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
except IndexError, KeyError:
    logging.warning(
        """Could not get all the required information (first image number and file template)
from the input file {0}""".format(input_file))

if last_image is not None and file_template is not None:
    # we have all the info to wait for the file

    #the file template is relative to the path passed to the script with the -path option
    file_template = os.path.abspath(os.path.join(path, file_template))
    fullpath = _template_to_image(file_template, int(last_image))
Beispiel #15
0
# parse the input file to find the first image
xds_appeared = False
wait_xds_start = time.time()
logging.debug('Waiting for XDS.INP file')
while not xds_appeared and time.time() - wait_xds_start < WAIT_XDS_TIMEOUT:
    if os.path.exists(input_file):
        xds_appeared = True
        logging.debug('XDS.INP file is there')
    else:
        time.sleep(WAIT_XDS_RESOLUTION)
if not xds_appeared:
    logging.error('XDS.INP file ({0}) failed to appear after {1} seconds'.format(input_file, WAIT_XDS_TIMEOUT))
    sys.exit(1)

cfg = xdscfgparser.parse_xds_file(input_file)
last_image = file_template = None
try:
    last_image = cfg['DATA_RANGE='][1]
    file_template = cfg['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
except IndexError, KeyError:
    logging.warning("""Could not get all the required information (first image number and file template)
from the input file {0}""".format(input_file))

if last_image is not None and file_template is not None:
    # we have all the info to wait for the file

    #the file template is relative to the path passed to the script with the -path option
    file_template = os.path.abspath(os.path.join(path, file_template))
    fullpath = _template_to_image(file_template, int(last_image))
    logging.info('Waiting for last frame {0}'.format(fullpath))
    def preProcess(self, _edObject=None):
        EDPluginExecProcessScript.preProcess(self)
        self.DEBUG("EDPluginMinimalXDS.preProcess")
        xds_input = os.path.abspath(self.dataInput.input_file.value)
        shutil.copy(xds_input, self.getWorkingDirectory())

        # our new xds file
        xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')

        parsed_config = parse_xds_file(xds_file)

        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]

        # Find out the real path for the template
        file_template_real_path = os.path.join(os.path.dirname(xds_input), file_template)

        # get the real directory the files are in by getting the real
        # path of the first image
        first_image_no = parsed_config['DATA_RANGE='][0]
        first_image = _template_to_image(file_template_real_path, first_image_no)

        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = file_template_real_path

        # perhaps modify some params
        job = self.dataInput.job
        maxproc = self.dataInput.maxproc
        maxjobs = self.dataInput.maxjobs
        resolution_range = self.dataInput.resolution_range
        friedels_law = self.dataInput.friedels_law
        spot_range = self.dataInput.spot_range
        spacegroup = self.dataInput.spacegroup
        unit_cell = self.dataInput.unit_cell

        self.DEBUG('requested spot range is {0}'.format(spot_range))

        if job is not None:
            parsed_config["JOB="] = job.value
        if maxproc is not None:
            parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
        if maxjobs is not None:
            parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
        if resolution_range is not None and len(resolution_range) != 0:
            parsed_config["INCLUDE_RESOLUTION_RANGE="] = [x.value for x in resolution_range]
        if friedels_law is not None:
            if friedels_law.value:
                parsed_config["FRIEDEL'S_LAW="] = "TRUE"
            else:
                parsed_config["FRIEDEL'S_LAW="] = "FALSE"
        if self.dataInput.start_image is not None:
            start_image = self.dataInput.start_image.value
            parsed_config['DATA_RANGE='][0] = start_image
        else:
            start_image = parsed_config['DATA_RANGE='][0]
        if self.dataInput.end_image is not None:
            end_image = self.dataInput.end_image.value
            parsed_config['DATA_RANGE='][1] = end_image
        else:
            end_image = parsed_config['DATA_RANGE='][1]
        spot_range_list = list()
        if spot_range is not None and len(spot_range) > 0:
            spot_range_list = []
            for srange in spot_range:
                spot_range_list.append([srange.begin, srange.end])
            spot_range_list = self.checkSpotRanges(spot_range_list, start_image, end_image)
        elif 'SPOT_RANGE=' in parsed_config:
            spot_range_list = self.checkSpotRanges(parsed_config['SPOT_RANGE='], start_image, end_image)
        else:
            spot_range_list = [[start_image, end_image]]
        self.DEBUG('setting the spot range to {0}'.format(spot_range_list))
        parsed_config['SPOT_RANGE='] = spot_range_list
        # Check background range
        if 'BACKGROUND_RANGE=' in parsed_config:
            background_range = parsed_config['BACKGROUND_RANGE=']
            if background_range[0] < start_image:
                background_range[0] = start_image
                background_range[1] += start_image - 1
            if background_range[1] > end_image:
                background_range[1] = end_image
            parsed_config['BACKGROUND_RANGE='] = background_range
        # unit cell might be an empty string or some other crazy stuff
        # we need 6 floats/ints
        if unit_cell is not None:
            ucells = unit_cell.value.split(' ')
            if len(ucells) != 6:
                unit_cell = None
            else:
                try:
                    if any(float(x) == 0 for x in ucells):
                        unit_cell = None
                except ValueError:
                    unit_cell = None
        # both need to be specified
        if spacegroup is not None and unit_cell is not None:
            self.DEBUG('specific spacegroup requested: {0}'.format(spacegroup.value))
            self.DEBUG('specific unit cell requested: {0}'.format(unit_cell.value))
            parsed_config['SPACE_GROUP_NUMBER='] = str(spacegroup.value)
            # Check if this is ok
            parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell.value

        # For [XY]-GEO_CORR files, link them in the cwd and fix their paths
        if 'X-GEO_CORR=' in parsed_config:
            xgeo = parsed_config['X-GEO_CORR='][0]
            xgeo_path = os.path.join(self.getWorkingDirectory(), os.path.basename(xgeo))
            if not os.path.exists(xgeo_path):
                os.symlink(xgeo, xgeo_path)
            parsed_config['X-GEO_CORR='] = os.path.basename(xgeo)
        if 'Y-GEO_CORR=' in parsed_config:
            ygeo = parsed_config['Y-GEO_CORR='][0]
            ygeo_path = os.path.join(self.getWorkingDirectory(), os.path.basename(ygeo))
            if not os.path.exists(ygeo_path):
                os.symlink(ygeo, ygeo_path)
            parsed_config['Y-GEO_CORR='] = os.path.basename(ygeo)

        # Max no processors
        parsed_config['MAXIMUM_NUMBER_OF_PROCESSORS='] = str(self.maxNoProcessors)
        parsed_config['MAXIMUM_NUMBER_OF_JOBS='] = 1

        # Save back the changes
        dump_xds_file(xds_file, parsed_config)
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlXdsBest.preProcess")
        self._xds = self.loadPlugin('EDPluginExecMinimalXdsv1_0')

        # save the root path (where the initial xds.inp is) for later use
        self.root_dir = os.path.abspath(
            os.path.dirname(self.dataInput.input_file.value))

        # let's begin by copying the input file to avoid clobbering it
        shutil.copy(self.dataInput.input_file.value,
                    self.getWorkingDirectory())
        self.real_input_file = os.path.join(
            self.getWorkingDirectory(),
            os.path.basename(self.dataInput.input_file.value))

        # update the keywords
        self.parsed_config = parse_xds_file(self.real_input_file)
        di = self.dataInput
        unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
            di.unit_cell_a.value, di.unit_cell_b.value, di.unit_cell_c.value,
            di.unit_cell_alpha.value, di.unit_cell_beta.value,
            di.unit_cell_gamma.value)
        sg = int(di.space_group.value)
        self.parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
        self.parsed_config['SPACE_GROUP_NUMBER='] = sg

        # to avoid any problem let's also make the image template
        # absolute as well. The underlying XDS template will take care
        # of creating the links in its dir
        imtemplate = self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        basedir = os.path.abspath(
            os.path.dirname(self.dataInput.input_file.value))
        newpath = os.path.join(basedir, imtemplate)
        self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in self.parsed_config:
            xgeo = os.path.abspath(
                os.path.join(self.root_dir,
                             self.parsed_config['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(xgeo))
                del self.parsed_config['X-GEO_CORR=']
            else:
                self.parsed_config['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in self.parsed_config:
            ygeo = os.path.abspath(
                os.path.join(self.root_dir,
                             self.parsed_config['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(ygeo))
                del self.parsed_config['Y-GEO_CORR=']
            else:
                self.parsed_config['Y-GEO_CORR='] = ygeo
        dump_xds_file(self.real_input_file, self.parsed_config)

        # create the input data model for the XDS plugin
        input_dm = XSDataMinimalXdsIn()
        input_dm.input_file = XSDataString(self.real_input_file)
        self._xds.dataInput = input_dm
    def preProcess(self, _edObject = None):
        EDPluginExecProcessScript.preProcess(self)
        self.DEBUG("EDPluginMinimalXDS.preProcess")
        xds_input = os.path.abspath(self.dataInput.input_file.value)
        shutil.copy(xds_input, self.getWorkingDirectory())

        # our new xds file
        xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')

        parsed_config = parse_xds_file(xds_file)

        # try to make some symlinks to work around the path length
        # limitation of xds
        # TODO: don't even try if not on Unix

        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]

        # get the real directory the files are in by getting the real
        # path of the first image
        first_image_no = parsed_config['DATA_RANGE='][0]
        first_image = _template_to_image(file_template, first_image_no)
        real_data_dir = os.path.dirname(os.path.realpath(first_image))

        # create a link to this dir in our dir - skip if path exists (i.e. multiple XDS runs)
        i_path = os.path.join(self.getWorkingDirectory(), 'i')
        if not os.path.exists(i_path):
            os.symlink(real_data_dir, i_path)

        # and update the config to refer to this dir
        new_template = os.path.join('i', os.path.basename(file_template))
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = new_template

        # perhaps modify some params
        job = self.dataInput.job
        maxproc = self.dataInput.maxproc
        maxjobs = self.dataInput.maxjobs
        resolution_range = self.dataInput.resolution_range
        friedels_law = self.dataInput.friedels_law
        spot_range = self.dataInput.spot_range
        spacegroup = self.dataInput.spacegroup
        unit_cell = self.dataInput.unit_cell

        self.DEBUG('requested spot range is {0}'.format(spot_range))

        if job is not None:
            parsed_config["JOB="] = job.value
        if maxproc is not None:
            parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
        if maxjobs is not None:
            parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
        if resolution_range is not None and len(resolution_range) != 0:
            parsed_config["INCLUDE_RESOLUTION_RANGE="] = [x.value for x in resolution_range]
        if friedels_law is not None:
            if friedels_law:
                parsed_config["FRIEDEL'S_LAW="] = "TRUE"
            else:
                parsed_config["FRIEDEL'S_LAW="] = "FALSE"
        if spot_range is not None and len(spot_range) > 0:
            spot_range_list = list()
            for srange in spot_range:
                spot_range_list.append('{0} {1}'.format(srange.begin, srange.end))
            self.DEBUG('setting the spot range to {0} as requested'.format(spot_range_list))
            parsed_config['SPOT_RANGE='] = spot_range_list
        # unit cell might be an empty string or some other crazy stuff
        # we need 6 floats/ints
        if unit_cell is not None:
            ucells = unit_cell.value.split()
            if len(ucells) != 6:
                unit_cell = None
            else:
                try:
                    if any(float(x) == 0 for x in ucells):
                        unit_cell = None
                except ValueError:
                    unit_cell = None
        # both need to be specified
        if spacegroup is not None and unit_cell is not None:
            self.DEBUG('specific spacegroup requested: {0}'.format(spacegroup.value))
            self.DEBUG('specific unit cell requested: {0}'.format(unit_cell.value))
            parsed_config['SPACE_GROUP_NUMBER='] = str(spacegroup.value)
            parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell.value

        # For [XY]-GEO_CORR files, link them in the cwd and fix their paths
        if 'X-GEO_CORR=' in parsed_config:
            xgeo = parsed_config['X-GEO_CORR='][0]
            xgeo_path = os.path.join(self.getWorkingDirectory(), os.path.basename(xgeo))
            if not os.path.exists(xgeo_path):
                os.symlink(xgeo, xgeo_path)
            parsed_config['X-GEO_CORR='] = os.path.basename(xgeo)
        if 'Y-GEO_CORR=' in parsed_config:
            ygeo = parsed_config['Y-GEO_CORR='][0]
            ygeo_path = os.path.join(self.getWorkingDirectory(), os.path.basename(ygeo))
            if not os.path.exists(ygeo_path):
                os.symlink(ygeo, ygeo_path)
            parsed_config['Y-GEO_CORR='] = os.path.basename(ygeo)

        # Save back the changes
        dump_xds_file(xds_file, parsed_config)
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG('EDPluginControlAutoproc.preProcess starting')
        self.DEBUG('failure state is currently {0}'.format(self.isFailure()))

        # for info to send to the autoproc stats server
        self.custom_stats = dict(creation_time=time.time(),
                                 processing_type='edna fastproc',
                                 datacollect_id=self.dataInput.data_collection_id.value,
                                 comments='running on {0}'.format(socket.gethostname()))


        data_in = self.dataInput
        xds_in = XSDataMinimalXdsIn()
        xds_in.input_file = data_in.input_file.path
        xds_in.spacegroup = data_in.spacegroup
        xds_in.unit_cell = data_in.unit_cell

        self.log_file_path = os.path.join(self.root_dir, 'stats.json')
        self.DEBUG('will log timing information to {0}'.format(self.log_file_path))
        self.stats = dict()

        # Get the image prefix from the directory name
        # XXX: This is horrible
        try:
            self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1])
        except Exception:
            self.image_prefix = ''

        self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing')
        try:
            os.makedirs(self.results_dir)
        except OSError: # it most likely exists
            pass

        # Copy the vanilla XDS input file to the results dir
        infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP')
        shutil.copy(self.dataInput.input_file.path.value,
                    infile_dest)

        # Ensure the autoproc ids directory is there
        self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids')
        try:
            os.makedirs(self.autoproc_ids_dir)
        except OSError: # it's there
            pass


        # we'll need the low res limit later on
        lowres = data_in.low_resolution_limit
        if lowres is not None:
            self.low_resolution_limit = lowres.value
        else:
            self.low_resolution_limit = 50

        res_override = data_in.res_override
        if res_override is not None:
            self.res_override = res_override.value
        else:
            # XXX: default to 0?
            self.res_override = None

        # check the number of images (must be > 8) and get the first
        # image name to wait for. Also modify the XDS.INP file to
        # reflect these values, if specified
        conf = parse_xds_file(data_in.input_file.path.value)


        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in conf:
            xgeo = os.path.abspath(os.path.join(self.root_dir,
                                                conf['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
                del conf['X-GEO_CORR=']
            else:
                conf['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in conf:
            ygeo = os.path.abspath(os.path.join(self.root_dir,
                                                conf['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
                del conf['Y-GEO_CORR=']
            else:
                conf['Y-GEO_CORR='] = ygeo

        dump_xds_file(data_in.input_file.path.value, conf)

        resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')

        if resrange is not None:
            if self.low_resolution_limit is not None:
                resrange[0] = self.low_resolution_limit
            if self.res_override is not None:
                resrange[1] = self.res_override
            conf['INCLUDE_RESOLUTION_RANGE='] = resrange
            dump_xds_file(data_in.input_file.path.value, conf)


        data_range = conf.get('DATA_RANGE=')
        # we'll need that for the very last part ( file import )
        self.data_range = data_range
        if data_range is not None:
            start_image = data_range[0]
            end_image = data_range[1]
            if end_image - start_image < 8:
                self.ERROR('there are fewer than 8 images, aborting')
                self.setFailure()
                return

        template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        self.DEBUG('template for images is {0}'.format(template))
        # fix the path if it's not absolute
        if not os.path.isabs(template):
            self.DEBUG('file template {0} is not absolute'.format(template))
            base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value))
            template = os.path.normpath(os.path.join(self.root_dir, template))
            conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template
            self.DEBUG('file template fixed to {0}'.format(template))
            self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value))
            dump_xds_file(data_in.input_file.path.value, conf)

        first_image = _template_to_image(template, start_image)


        self.wait_file = self.loadPlugin('EDPluginWaitFile')
        waitfileinput = XSDataInputWaitFile()
        waitfileinput.expectedFile = XSDataFile()
        waitfileinput.expectedFile.path = XSDataString(first_image)
        waitfileinput.expectedSize = XSDataInteger(0) # we do not care
        timeout = XSDataTime()
        global WAIT_FOR_FRAME_TIMEOUT
        timeout.value = WAIT_FOR_FRAME_TIMEOUT
        waitfileinput.timeOut = timeout
        self.wait_file.dataInput = waitfileinput

        self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
        self.xds_first.dataInput = xds_in

        self.generate = self.loadPlugin("EDPluginXDSGenerate")

        self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")

        self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
        self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")

        self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")

        self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
        self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')

        self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')

        self.DEBUG('EDPluginControlAutoproc.preProcess finished')
Beispiel #20
0
    def preProcess(self, _edObject=None):
        EDPluginExecProcessScript.preProcess(self)
        self.DEBUG("EDPluginMinimalXDS.preProcess")
        xds_input = os.path.abspath(self.dataInput.input_file.value)
        shutil.copy(xds_input, self.getWorkingDirectory())

        # our new xds file
        xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')

        parsed_config = parse_xds_file(xds_file)

        # try to make some symlinks to work around the path length
        # limitation of xds
        # TODO: don't even try if not on Unix

        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]

        # get the real directory the files are in by getting the real
        # path of the first image
        first_image_no = parsed_config['DATA_RANGE='][0]
        first_image = _template_to_image(file_template, first_image_no)
        real_data_dir = os.path.dirname(os.path.realpath(first_image))

        # create a link to this dir in our dir - skip if path exists (i.e. multiple XDS runs)
        i_path = os.path.join(self.getWorkingDirectory(), 'i')
        if not os.path.exists(i_path):
            os.symlink(real_data_dir, i_path)

        # and update the config to refer to this dir
        new_template = os.path.join('i', os.path.basename(file_template))
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = new_template

        # perhaps modify some params
        job = self.dataInput.job
        maxproc = self.dataInput.maxproc
        maxjobs = self.dataInput.maxjobs
        resolution_range = self.dataInput.resolution_range
        friedels_law = self.dataInput.friedels_law
        spot_range = self.dataInput.spot_range
        spacegroup = self.dataInput.spacegroup
        unit_cell = self.dataInput.unit_cell

        self.DEBUG('requested spot range is {0}'.format(spot_range))

        if job is not None:
            parsed_config["JOB="] = job.value
        if maxproc is not None:
            parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
        if maxjobs is not None:
            parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
        if resolution_range is not None and len(resolution_range) != 0:
            parsed_config["INCLUDE_RESOLUTION_RANGE="] = [
                x.value for x in resolution_range
            ]
        if friedels_law is not None:
            if friedels_law:
                parsed_config["FRIEDEL'S_LAW="] = "TRUE"
            else:
                parsed_config["FRIEDEL'S_LAW="] = "FALSE"
        if spot_range is not None and len(spot_range) > 0:
            spot_range_list = list()
            for srange in spot_range:
                spot_range_list.append('{0} {1}'.format(
                    srange.begin, srange.end))
            self.DEBUG('setting the spot range to {0} as requested'.format(
                spot_range_list))
            parsed_config['SPOT_RANGE='] = spot_range_list
        # unit cell might be an empty string or some other crazy stuff
        # we need 6 floats/ints
        if unit_cell is not None:
            ucells = unit_cell.value.split()
            if len(ucells) != 6:
                unit_cell = None
            else:
                try:
                    if any(float(x) == 0 for x in ucells):
                        unit_cell = None
                except ValueError:
                    unit_cell = None
        # both need to be specified
        if spacegroup is not None and unit_cell is not None:
            self.DEBUG('specific spacegroup requested: {0}'.format(
                spacegroup.value))
            self.DEBUG('specific unit cell requested: {0}'.format(
                unit_cell.value))
            parsed_config['SPACE_GROUP_NUMBER='] = str(spacegroup.value)
            parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell.value

        # For [XY]-GEO_CORR files, link them in the cwd and fix their paths
        if 'X-GEO_CORR=' in parsed_config:
            xgeo = parsed_config['X-GEO_CORR='][0]
            xgeo_path = os.path.join(self.getWorkingDirectory(),
                                     os.path.basename(xgeo))
            if not os.path.exists(xgeo_path):
                os.symlink(xgeo, xgeo_path)
            parsed_config['X-GEO_CORR='] = os.path.basename(xgeo)
        if 'Y-GEO_CORR=' in parsed_config:
            ygeo = parsed_config['Y-GEO_CORR='][0]
            ygeo_path = os.path.join(self.getWorkingDirectory(),
                                     os.path.basename(ygeo))
            if not os.path.exists(ygeo_path):
                os.symlink(ygeo, ygeo_path)
            parsed_config['Y-GEO_CORR='] = os.path.basename(ygeo)

        # Save back the changes
        dump_xds_file(xds_file, parsed_config)
Beispiel #21
0
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG('EDPluginControlAutoproc.preProcess starting')
        self.DEBUG('failure state is currently {0}'.format(self.isFailure()))

        # for info to send to the autoproc stats server
        self.custom_stats = dict(
            creation_time=time.time(),
            processing_type='edna fastproc',
            datacollect_id=self.dataInput.data_collection_id.value,
            comments='running on {0}'.format(socket.gethostname()))

        data_in = self.dataInput
        xds_in = XSDataMinimalXdsIn()
        xds_in.input_file = data_in.input_file.path
        xds_in.spacegroup = data_in.spacegroup
        xds_in.unit_cell = data_in.unit_cell

        self.log_file_path = os.path.join(self.root_dir, 'stats.json')
        self.DEBUG('will log timing information to {0}'.format(
            self.log_file_path))
        self.stats = dict()

        # Get the image prefix from the directory name
        # XXX: This is horrible
        try:
            self.image_prefix = '_'.join(
                os.path.basename(self.root_dir).split('_')[1:-1])
        except Exception:
            self.image_prefix = ''

        self.results_dir = os.path.join(self.root_dir, 'results',
                                        'fast_processing')
        try:
            os.makedirs(self.results_dir)
        except OSError:  # it most likely exists
            pass

        # Copy the vanilla XDS input file to the results dir
        infile_dest = os.path.join(self.results_dir,
                                   self.image_prefix + '_input_XDS.INP')
        shutil.copy(self.dataInput.input_file.path.value, infile_dest)

        # Ensure the autoproc ids directory is there
        self.autoproc_ids_dir = os.path.join(self.results_dir,
                                             'fastproc_integration_ids')
        try:
            os.makedirs(self.autoproc_ids_dir)
        except OSError:  # it's there
            pass

        # we'll need the low res limit later on
        lowres = data_in.low_resolution_limit
        if lowres is not None:
            self.low_resolution_limit = lowres.value
        else:
            self.low_resolution_limit = 50

        res_override = data_in.res_override
        if res_override is not None:
            self.res_override = res_override.value
        else:
            # XXX: default to 0?
            self.res_override = None

        # check the number of images (must be > 8) and get the first
        # image name to wait for. Also modify the XDS.INP file to
        # reflect these values, if specified
        conf = parse_xds_file(data_in.input_file.path.value)

        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in conf:
            xgeo = os.path.abspath(
                os.path.join(self.root_dir, conf['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(xgeo))
                del conf['X-GEO_CORR=']
            else:
                conf['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in conf:
            ygeo = os.path.abspath(
                os.path.join(self.root_dir, conf['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(ygeo))
                del conf['Y-GEO_CORR=']
            else:
                conf['Y-GEO_CORR='] = ygeo

        dump_xds_file(data_in.input_file.path.value, conf)

        resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')

        if resrange is not None:
            if self.low_resolution_limit is not None:
                resrange[0] = self.low_resolution_limit
            if self.res_override is not None:
                resrange[1] = self.res_override
            conf['INCLUDE_RESOLUTION_RANGE='] = resrange
            dump_xds_file(data_in.input_file.path.value, conf)

        data_range = conf.get('DATA_RANGE=')
        # we'll need that for the very last part ( file import )
        self.data_range = data_range
        if data_range is not None:
            start_image = data_range[0]
            end_image = data_range[1]
            if end_image - start_image < 8:
                self.ERROR('there are fewer than 8 images, aborting')
                self.setFailure()
                return

        template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        self.DEBUG('template for images is {0}'.format(template))
        # fix the path if it's not absolute
        if not os.path.isabs(template):
            self.DEBUG('file template {0} is not absolute'.format(template))
            base_dir = os.path.abspath(
                os.path.dirname(data_in.input_file.path.value))
            template = os.path.normpath(os.path.join(self.root_dir, template))
            conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template
            self.DEBUG('file template fixed to {0}'.format(template))
            self.DEBUG('dumping back the file to {0}'.format(
                data_in.input_file.path.value))
            dump_xds_file(data_in.input_file.path.value, conf)

        first_image = _template_to_image(template, start_image)

        self.wait_file = self.loadPlugin('EDPluginWaitFile')
        waitfileinput = XSDataInputWaitFile()
        waitfileinput.expectedFile = XSDataFile()
        waitfileinput.expectedFile.path = XSDataString(first_image)
        waitfileinput.expectedSize = XSDataInteger(0)  # we do not care
        timeout = XSDataTime()
        global WAIT_FOR_FRAME_TIMEOUT
        timeout.value = WAIT_FOR_FRAME_TIMEOUT
        waitfileinput.timeOut = timeout
        self.wait_file.dataInput = waitfileinput

        self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
        self.xds_first.dataInput = xds_in

        self.generate = self.loadPlugin("EDPluginXDSGenerate")

        self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")

        self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
        self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")

        self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")

        self.store_autoproc_anom = self.loadPlugin(
            'EDPluginISPyBStoreAutoProcv1_4')
        self.store_autoproc_noanom = self.loadPlugin(
            'EDPluginISPyBStoreAutoProcv1_4')

        self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')

        self.DEBUG('EDPluginControlAutoproc.preProcess finished')
    def preProcess(self, _edObject=None):
        EDPluginExecProcessScript.preProcess(self)
        self.DEBUG("EDPluginMinimalXDS.preProcess")
        xds_input = os.path.abspath(self.dataInput.input_file.value)
        workingDirectory = self.getWorkingDirectory()
        if not os.path.exists(workingDirectory):
            os.makedirs(workingDirectory, 0o755)
        shutil.copy(xds_input, workingDirectory)

        # our new xds file
        xds_file = os.path.join(self.getWorkingDirectory(), 'XDS.INP')

        parsed_config = parse_xds_file(xds_file)

        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]

        # Find out the real path for the template
        file_template_real_path = os.path.join(os.path.dirname(xds_input),
                                               file_template)

        # get the real directory the files are in by getting the real
        # path of the first image
        first_image_no = parsed_config['DATA_RANGE='][0]
        first_image = _template_to_image(file_template_real_path,
                                         first_image_no)

        parsed_config[
            'NAME_TEMPLATE_OF_DATA_FRAMES='] = file_template_real_path

        # perhaps modify some params
        job = self.dataInput.job
        maxproc = self.dataInput.maxproc
        maxjobs = self.dataInput.maxjobs
        resolution_range = self.dataInput.resolution_range
        friedels_law = self.dataInput.friedels_law
        spot_range = self.dataInput.spot_range
        spacegroup = self.dataInput.spacegroup
        unit_cell = self.dataInput.unit_cell

        self.DEBUG('requested spot range is {0}'.format(spot_range))

        if job is not None:
            parsed_config["JOB="] = job.value
        if maxproc is not None:
            parsed_config["MAXIMUM_NUMBER_OF_PROCESSORS="] = maxproc.value
        if maxjobs is not None:
            parsed_config["MAXIMUM_NUMBER_OF_JOBS="] = maxjobs.value
        if resolution_range is not None and len(resolution_range) != 0:
            parsed_config["INCLUDE_RESOLUTION_RANGE="] = [
                x.value for x in resolution_range
            ]
        if friedels_law is not None:
            if friedels_law.value:
                parsed_config["FRIEDEL'S_LAW="] = "TRUE"
            else:
                parsed_config["FRIEDEL'S_LAW="] = "FALSE"
        if self.dataInput.start_image is not None:
            start_image = self.dataInput.start_image.value
            parsed_config['DATA_RANGE='][0] = start_image
        else:
            start_image = parsed_config['DATA_RANGE='][0]
        if self.dataInput.end_image is not None:
            end_image = self.dataInput.end_image.value
            parsed_config['DATA_RANGE='][1] = end_image
        else:
            end_image = parsed_config['DATA_RANGE='][1]
        spot_range_list = list()
        if spot_range is not None and len(spot_range) > 0:
            spot_range_list = []
            for srange in spot_range:
                spot_range_list.append([srange.begin, srange.end])
            spot_range_list = self.checkSpotRanges(spot_range_list,
                                                   start_image, end_image)
        elif 'SPOT_RANGE=' in parsed_config:
            spot_range_list = self.checkSpotRanges(
                parsed_config['SPOT_RANGE='], start_image, end_image)
        else:
            spot_range_list = [[start_image, end_image]]
        self.DEBUG('setting the spot range to {0}'.format(spot_range_list))
        parsed_config['SPOT_RANGE='] = spot_range_list
        # Check background range
        if 'BACKGROUND_RANGE=' in parsed_config:
            background_range = parsed_config['BACKGROUND_RANGE=']
            if background_range[0] < start_image:
                background_range[0] = start_image
                background_range[1] += start_image - 1
            if background_range[1] > end_image:
                background_range[1] = end_image
            parsed_config['BACKGROUND_RANGE='] = background_range
        # unit cell might be an empty string or some other crazy stuff
        # we need 6 floats/ints
        if unit_cell is not None:
            ucells = unit_cell.value.split(' ')
            if len(ucells) != 6:
                unit_cell = None
            else:
                try:
                    if any(float(x) == 0 for x in ucells):
                        unit_cell = None
                except ValueError:
                    unit_cell = None
        # both need to be specified
        if spacegroup is not None and unit_cell is not None:
            self.DEBUG('specific spacegroup requested: {0}'.format(
                spacegroup.value))
            self.DEBUG('specific unit cell requested: {0}'.format(
                unit_cell.value))
            parsed_config['SPACE_GROUP_NUMBER='] = str(spacegroup.value)
            # Check if this is ok
            parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell.value

        # For [XY]-GEO_CORR files, link them in the cwd and fix their paths
        if 'X-GEO_CORR=' in parsed_config:
            xgeo = parsed_config['X-GEO_CORR='][0]
            xgeo_path = os.path.join(self.getWorkingDirectory(),
                                     os.path.basename(xgeo))
            if not os.path.exists(xgeo_path):
                os.symlink(xgeo, xgeo_path)
            parsed_config['X-GEO_CORR='] = os.path.basename(xgeo)
        if 'Y-GEO_CORR=' in parsed_config:
            ygeo = parsed_config['Y-GEO_CORR='][0]
            ygeo_path = os.path.join(self.getWorkingDirectory(),
                                     os.path.basename(ygeo))
            if not os.path.exists(ygeo_path):
                os.symlink(ygeo, ygeo_path)
            parsed_config['Y-GEO_CORR='] = os.path.basename(ygeo)

        # Neggia plugin
        if not "LIB=" in parsed_config and self.pathToNeggiaPlugin is not None:
            parsed_config["LIB="] = self.pathToNeggiaPlugin

        # Max no processors
        if not EDUtilsPath.isEMBL():
            parsed_config['MAXIMUM_NUMBER_OF_PROCESSORS='] = str(
                self.maxNoProcessors)
        parsed_config['MAXIMUM_NUMBER_OF_JOBS='] = 1

        # Save back the changes
        dump_xds_file(xds_file, parsed_config)