def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlXdsBest.preProcess")
        self._xds = self.loadPlugin('EDPluginExecMinimalXds')

        # save the root path (where the initial xds.inp is) for later use
        self.root_dir = os.path.abspath(os.path.dirname(self.dataInput.input_file.value))

        # let's begin by copying the input file to avoid clobbering it
        shutil.copy(self.dataInput.input_file.value,
                    self.getWorkingDirectory())
        self.real_input_file = os.path.join(self.getWorkingDirectory(),
                                       os.path.basename(self.dataInput.input_file.value))

        # update the keywords
        self.parsed_config = parse_xds_file(self.real_input_file)
        di = self.dataInput
        unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
            di.unit_cell_a.value,
            di.unit_cell_b.value,
            di.unit_cell_c.value,
            di.unit_cell_alpha.value,
            di.unit_cell_beta.value,
            di.unit_cell_gamma.value)
        sg = int(di.space_group.value)
        self.parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
        self.parsed_config['SPACE_GROUP_NUMBER='] = sg

        # to avoid any problem let's also make the image template
        # absolute as well. The underlying XDS template will take care
        # of creating the links in its dir
        imtemplate = self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        basedir = os.path.abspath(os.path.dirname(self.dataInput.input_file.value))
        newpath = os.path.join(basedir, imtemplate)
        self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in self.parsed_config:
            xgeo = os.path.abspath(os.path.join(self.root_dir,
                                                self.parsed_config['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
                del self.parsed_config['X-GEO_CORR=']
            else:
                self.parsed_config['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in self.parsed_config:
            ygeo = os.path.abspath(os.path.join(self.root_dir,
                                                self.parsed_config['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
                del self.parsed_config['Y-GEO_CORR=']
            else:
                self.parsed_config['Y-GEO_CORR='] = ygeo
        dump_xds_file(self.real_input_file, self.parsed_config)

        # create the input data model for the XDS plugin
        input_dm = XSDataMinimalXdsIn()
        input_dm.input_file = XSDataString(self.real_input_file)
        self._xds.dataInput = input_dm
 def process(self, _edObject = None):
     EDPluginControl.process(self)
     self.DEBUG("EDPluginControlXdsBest.process")
     self._xds.executeSynchronous()
     expected = os.path.join(self._xds.getWorkingDirectory(), 'XDS_ASCII.HKL')
     if not os.path.exists(expected):
         # Try to re-run with JOB = DEFPIX INTEGRATE CORRECT
         self.parsed_config["JOB="] = "DEFPIX INTEGRATE CORRECT"
         self._xds2 = self.loadPlugin('EDPluginExecMinimalXds')
         self._xds2.setWorkingDirectory(self._xds.getWorkingDirectory())
         dump_xds_file(self.real_input_file, self.parsed_config)
         input_dm2 = XSDataMinimalXdsIn()
         input_dm2.input_file = XSDataString(self.real_input_file)
         self._xds2.dataInput = input_dm2
         self._xds2.executeSynchronous()
 def process(self, _edObject=None):
     EDPluginControl.process(self)
     self.DEBUG("EDPluginControlXdsBest.process")
     self._xds.executeSynchronous()
     expected = os.path.join(self._xds.getWorkingDirectory(),
                             'XDS_ASCII.HKL')
     if not os.path.exists(expected):
         # Try to re-run with JOB = DEFPIX INTEGRATE CORRECT
         self.parsed_config["JOB="] = "DEFPIX INTEGRATE CORRECT"
         self._xds2 = self.loadPlugin('EDPluginExecMinimalXds')
         self._xds2.setWorkingDirectory(self._xds.getWorkingDirectory())
         dump_xds_file(self.real_input_file, self.parsed_config)
         input_dm2 = XSDataMinimalXdsIn()
         input_dm2.input_file = XSDataString(self.real_input_file)
         self._xds2.dataInput = input_dm2
         self._xds2.executeSynchronous()
示例#4
0
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAutoproc.preProcess")

        data_in = self.dataInput
        xds_in = XSDataMinimalXdsIn()
        xds_in.input_file = data_in.input_file.path

        self.xds_first = self.loadPlugin("EDPluginControlRunXds")
        self.xds_first.dataInput = xds_in

        self.generate = self.loadPlugin("EDPluginXDSGenerate")

        self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")

        self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
        self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")

        self.xscale_anom = self.loadPlugin("EDPluginControlXscaleGenerate")
        self.xscale_noanom = self.loadPlugin("EDPluginControlXscaleGenerate")
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlXdsBest.preProcess")
        self._xds = self.loadPlugin('EDPluginExecMinimalXds')

        # let's begin by copying the input file to avoid clobbering it
        shutil.copy(self.dataInput.input_file.value,
                    self.getWorkingDirectory())
        real_input_file = os.path.join(self.getWorkingDirectory(),
                                       os.path.basename(self.dataInput.input_file.value))
        # update the keywords
        parsed_config = parse_xds_file(real_input_file)
        di = self.dataInput
        unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
            di.unit_cell_a.value,
            di.unit_cell_b.value,
            di.unit_cell_c.value,
            di.unit_cell_alpha.value,
            di.unit_cell_beta.value,
            di.unit_cell_gamma.value)
        sg = int(di.space_group.value)
        parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
        parsed_config['SPACE_GROUP_NUMBER='] = sg

        # to avoid any problem let's also make the image template
        # absolute as well. The underlying XDS template will take care
        # of creating the links in its dir
        imtemplate = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        basedir = os.path.abspath(self.dataInput.input_file.value)
        newpath = os.path.join(basedir, imtemplate)
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath

        dump_xds_file(real_input_file, parsed_config)

        # create the input data model for the XDS plugin
        input_dm = XSDataMinimalXdsIn()
        input_dm.input_file = XSDataString(real_input_file)
        self._xds.dataInput = input_dm
示例#6
0
    def process(self, _edObject = None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlRunXds.process")
        # First run is vanilla without any modification
        params = XSDataMinimalXdsIn()
        params.input_file = self.dataInput.input_file
        self.first_run.dataInput = params
        self.first_run.executeSynchronous()

        EDVerbose.DEBUG('first run completed...')

        if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
            EDVerbose.DEBUG('... and it worked')
            self.successful_run = self.first_run
        else:
            EDVerbose.DEBUG('... and it failed')


        if not self.successful_run:
            # second run w/ JOB set to DEFPIX INTEGRATE CORRECT
            self.second_run = self.loadPlugin(self.controlled_plugin_name)
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.jobs = 'DEFPIX INTEGRATE CORRECT'
            self.second_run.dataInput = params
            self.second_run.executeSynchronous()

            EDVerbose.DEBUG('second run completed')
            if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.second_run
            else:
                EDVerbose.DEBUG('... and it failed')


        if not self.successful_run:
        # third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1
            self.third_run = self.loadPlugin(self.controlled_plugin_name)
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.jobs = 'ALL'
            params.maxprocs = 4
            params.maxjobs = 1
            self.third_run.dataInput = params
            self.third_run.executeSynchronous()

            EDVerbose.DEBUG('third run completed')
            if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.third_run
            else:
                EDVerbose.DEBUG('... and it failed')


        if not self.successful_run:
        # final run with parallelism like 3 but JOB like 2
            self.final_run = self.loadPlugin(self.controlled_plugin_name)
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.jobs = 'DEFPIX INTEGRATE CORRECT'
            params.maxprocs = 4
            params.maxjobs = 1
            self.final_run.dataInput = params
            self.final_run.executeSynchronous()

            EDVerbose.DEBUG('final run completed')
            if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.final_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
        # all runs failed so bail out ...
            self.setFailure()
        else:
            # use the xds parser plugin to parse the xds output file...
            parser = self.loadPlugin("EDPluginParseXdsOutput")
            wd = self.successful_run.getWorkingDirectory()
            parser_input = XSDataXdsOutputFile()
            correct_lp_path = XSDataFile()
            correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
            parser_input.correct_lp = correct_lp_path
            gxparm_path = os.path.join(wd, 'GXPARM.XDS')
            if os.path.isfile(gxparm_path):
                gxparm = XSDataFile()
                gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
                parser_input.gxparm = gxparm

            parser.dataInput = parser_input
            parser.executeSynchronous()

            if parser.isFailure():
                # that should not happen
                self.setFailure()
                return
            self.dataOutput = parser.dataOutput
示例#7
0
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAutoproc.preProcess")

        self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
        self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')

        path = os.path.abspath(self.dataInput.previous_run_dir.value)

        # create the data inputs now we know the files are here
        input_anom = XSDataMinimalXdsIn()
        input_anom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
        input_anom.friedels_law = XSDataBoolean(True)
        input_anom.job = XSDataString('CORRECT')
        input_anom.resolution = self.dataInput.resolution
        input_anom.resolution_range = [
            XSDataFloat(60), self.dataInput.resolution
        ]
        self.xds_anom.dataInput = input_anom

        input_noanom = XSDataMinimalXdsIn()
        input_noanom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
        input_noanom.fridels_law = XSDataBoolean(False)
        input_noanom.job = XSDataString('CORRECT')
        input_noanom.resolution_range = [
            XSDataFloat(60), self.dataInput.resolution
        ]
        self.xds_noanom.dataInput = input_noanom

        xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
        xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())
        # let's make some links!
        for f in self._to_link:
            os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
            os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
        # now this is the horrible part, we need to make symlinks to
        # the images also. for now we rely on the fact that the links
        # in the previous run are most likely the links to the
        # images. So we will make the same links and rely on the fact
        # that in the input file the path to the images is already
        # relative to the CWD
        path = os.path.abspath(self.dataInput.previous_run_dir.value)
        for f in os.listdir(path):
            fullpath = os.path.join(path, f)
            if os.path.islink(fullpath):
                # symlink the symlink...
                os.symlink(fullpath, os.path.join(xds_anom_dir, f))
                os.symlink(fullpath, os.path.join(xds_noanom_dir, f))
    def process(self, _edObject = None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlRunXdsFastProc.process")
        # First run is vanilla without any modification
        params = XSDataMinimalXdsIn()
        params.input_file = self.dataInput.input_file
        params.spacegroup = self.dataInput.spacegroup
        params.unit_cell = self.dataInput.unit_cell
        self.first_run.dataInput = params
        self.first_run.executeSynchronous()

        EDVerbose.DEBUG('first run completed...')

        if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
            EDVerbose.DEBUG('... and it worked')
            self.successful_run = self.first_run
        else:
            EDVerbose.DEBUG('... and it failed')


        if not self.successful_run:
            self.second_run = self.loadPlugin(self.controlled_plugin_name)
            self.DEBUG('retrying with increased SPOT_RANGE')
            self.DEBUG('copying previously generated files to the new plugin dir')
            copy_xds_files(self.first_run.getWorkingDirectory(),
                           self.second_run.getWorkingDirectory())
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
            params.spacegroup = self.dataInput.spacegroup
            params.unit_cell = self.dataInput.unit_cell

            # increase all the spot ranges end by 20, constrained to
            # the data range upper limit
            spot_range = list()
            for srange in self.spot_range:
                range_begin = srange[0]
                range_end = srange[1] + 20
                if range_end > self.end_image_no:
                    self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
                                                                                          self.end_image_no))
                    range_end = self.end_image_no
                self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
                r = XSDataRange(begin=range_begin, end=range_end)
                spot_range.append(r)

            params.spot_range = spot_range

            self.second_run.dataInput = params
            self.second_run.executeSynchronous()

            EDVerbose.DEBUG('second run completed')
            if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.second_run
            else:
                EDVerbose.DEBUG('... and it failed')


        if not self.successful_run:
            self.third_run = self.loadPlugin(self.controlled_plugin_name)
            self.DEBUG('retrying with increased SPOT_RANGE')
            self.DEBUG('copying previously generated files to the new plugin dir')
            copy_xds_files(self.second_run.getWorkingDirectory(),
                           self.third_run.getWorkingDirectory())
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
            params.spacegroup = self.dataInput.spacegroup
            params.unit_cell = self.dataInput.unit_cell
            spot_range = list()
            for srange in self.spot_range:
                range_begin = srange[0]
                range_end = srange[1] + 40
                if range_end > self.end_image_no:
                    self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end,
                                                                                          self.end_image_no))
                    range_end = self.end_image_no
                self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end))
                r = XSDataRange(begin=range_begin, end=range_end)
                spot_range.append(r)

            params.spot_range = spot_range

            self.third_run.dataInput = params
            self.third_run.executeSynchronous()

            EDVerbose.DEBUG('third run completed')
            if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.third_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
        # all runs failed so bail out ...
            self.setFailure()
        else:
            # use the xds parser plugin to parse the xds output file...
            parser = self.loadPlugin("EDPluginParseXdsOutput")
            wd = self.successful_run.getWorkingDirectory()
            parser_input = XSDataXdsOutputFile()
            correct_lp_path = XSDataFile()
            correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
            parser_input.correct_lp = correct_lp_path
            gxparm_path = os.path.join(wd, 'GXPARM.XDS')
            if os.path.isfile(gxparm_path):
                gxparm = XSDataFile()
                gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
                parser_input.gxparm = gxparm

            parser.dataInput = parser_input
            parser.executeSynchronous()

            if parser.isFailure():
                # that should not happen
                self.setFailure()
                return
            self.dataOutput = parser.dataOutput
示例#9
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlRunXdsFastProc.process")
        # First run is vanilla without any modification
        params = XSDataMinimalXdsIn()
        params.input_file = self.dataInput.input_file
        params.spacegroup = self.dataInput.spacegroup
        params.unit_cell = self.dataInput.unit_cell
        self.first_run.dataInput = params
        self.first_run.executeSynchronous()

        EDVerbose.DEBUG('first run completed...')

        if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value:
            EDVerbose.DEBUG('... and it worked')
            self.successful_run = self.first_run
        else:
            EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            self.second_run = self.loadPlugin(self.controlled_plugin_name)
            self.DEBUG('retrying with increased SPOT_RANGE')
            self.DEBUG(
                'copying previously generated files to the new plugin dir')
            copy_xds_files(self.first_run.getWorkingDirectory(),
                           self.second_run.getWorkingDirectory())
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
            params.spacegroup = self.dataInput.spacegroup
            params.unit_cell = self.dataInput.unit_cell

            # increase all the spot ranges end by 20, constrained to
            # the data range upper limit
            spot_range = list()
            for srange in self.spot_range:
                range_begin = srange[0]
                range_end = srange[1] + 20
                if range_end > self.end_image_no:
                    self.DEBUG(
                        'End of range {0} would be past the last image {1}'.
                        format(range_end, self.end_image_no))
                    range_end = self.end_image_no
                self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
                    srange, range_begin, range_end))
                r = XSDataRange(begin=range_begin, end=range_end)
                spot_range.append(r)

            params.spot_range = spot_range

            self.second_run.dataInput = params
            self.second_run.executeSynchronous()

            EDVerbose.DEBUG('second run completed')
            if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.second_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            self.third_run = self.loadPlugin(self.controlled_plugin_name)
            self.DEBUG('retrying with increased SPOT_RANGE')
            self.DEBUG(
                'copying previously generated files to the new plugin dir')
            copy_xds_files(self.second_run.getWorkingDirectory(),
                           self.third_run.getWorkingDirectory())
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.job = XSDataString('DEFPIX INTEGRATE CORRECT')
            params.spacegroup = self.dataInput.spacegroup
            params.unit_cell = self.dataInput.unit_cell
            spot_range = list()
            for srange in self.spot_range:
                range_begin = srange[0]
                range_end = srange[1] + 40
                if range_end > self.end_image_no:
                    self.DEBUG(
                        'End of range {0} would be past the last image {1}'.
                        format(range_end, self.end_image_no))
                    range_end = self.end_image_no
                self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(
                    srange, range_begin, range_end))
                r = XSDataRange(begin=range_begin, end=range_end)
                spot_range.append(r)

            params.spot_range = spot_range

            self.third_run.dataInput = params
            self.third_run.executeSynchronous()

            EDVerbose.DEBUG('third run completed')
            if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.third_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            # all runs failed so bail out ...
            self.setFailure()
        else:
            # use the xds parser plugin to parse the xds output file...
            parser = self.loadPlugin("EDPluginParseXdsOutput")
            wd = self.successful_run.getWorkingDirectory()
            parser_input = XSDataXdsOutputFile()
            correct_lp_path = XSDataFile()
            correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
            parser_input.correct_lp = correct_lp_path
            gxparm_path = os.path.join(wd, 'GXPARM.XDS')
            if os.path.isfile(gxparm_path):
                gxparm = XSDataFile()
                gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
                parser_input.gxparm = gxparm

            parser.dataInput = parser_input
            parser.executeSynchronous()

            if parser.isFailure():
                # that should not happen
                self.setFailure()
                return
            self.dataOutput = parser.dataOutput
示例#10
0
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlRunXds.process")
        # First run is vanilla without any modification
        params = XSDataMinimalXdsIn()
        params.input_file = self.dataInput.input_file
        self.first_run.dataInput = params
        self.first_run.executeSynchronous()

        EDVerbose.DEBUG('first run completed...')

        if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded:
            EDVerbose.DEBUG('... and it worked')
            self.successful_run = self.first_run
        else:
            EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            # second run w/ JOB set to DEFPIX INTEGRATE CORRECT
            self.second_run = self.loadPlugin(self.controlled_plugin_name)
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.jobs = 'DEFPIX INTEGRATE CORRECT'
            self.second_run.dataInput = params
            self.second_run.executeSynchronous()

            EDVerbose.DEBUG('second run completed')
            if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.second_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            # third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1
            self.third_run = self.loadPlugin(self.controlled_plugin_name)
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.jobs = 'ALL'
            params.maxprocs = 4
            params.maxjobs = 1
            self.third_run.dataInput = params
            self.third_run.executeSynchronous()

            EDVerbose.DEBUG('third run completed')
            if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.third_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            # final run with parallelism like 3 but JOB like 2
            self.final_run = self.loadPlugin(self.controlled_plugin_name)
            params = XSDataMinimalXdsIn()
            params.input_file = self.dataInput.input_file
            params.jobs = 'DEFPIX INTEGRATE CORRECT'
            params.maxprocs = 4
            params.maxjobs = 1
            self.final_run.dataInput = params
            self.final_run.executeSynchronous()

            EDVerbose.DEBUG('final run completed')
            if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded:
                EDVerbose.DEBUG('... and it worked')
                self.successful_run = self.final_run
            else:
                EDVerbose.DEBUG('... and it failed')

        if not self.successful_run:
            # all runs failed so bail out ...
            self.setFailure()
        else:
            # use the xds parser plugin to parse the xds output file...
            parser = self.loadPlugin("EDPluginParseXdsOutput")
            wd = self.successful_run.getWorkingDirectory()
            parser_input = XSDataXdsOutputFile()
            correct_lp_path = XSDataFile()
            correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP'))
            parser_input.correct_lp = correct_lp_path
            gxparm_path = os.path.join(wd, 'GXPARM.XDS')
            if os.path.isfile(gxparm_path):
                gxparm = XSDataFile()
                gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS'))
                parser_input.gxparm = gxparm

            parser.dataInput = parser_input
            parser.executeSynchronous()

            if parser.isFailure():
                # that should not happen
                self.setFailure()
                return
            self.dataOutput = parser.dataOutput
示例#11
0
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAutoproc.preProcess")

        self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
        self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')

        path = os.path.abspath(self.dataInput.previous_run_dir.value)

        # The MinimalXds plugin takes care of creating determining the
        # real images directory and creating a symlink to it so we
        # only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
        # to not be relative anymore. We'll copy it to our own dir
        # beforehand to avoid clobbering it
        xdsinp = os.path.join(path, 'XDS.INP')
        new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
        copyfile(xdsinp, new_xdsinp)


        parsed_config = parse_xds_file(new_xdsinp)
        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(os.path.join(path, file_template))
        dump_xds_file(new_xdsinp, parsed_config)

        # create the data inputs now we know the files are here
        input_anom = XSDataMinimalXdsIn()
        input_anom.input_file = XSDataString(new_xdsinp)
        input_anom.friedels_law = XSDataBoolean(False)
        input_anom.job = XSDataString('CORRECT')
        input_anom.resolution = self.dataInput.resolution
        input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_anom.dataInput = input_anom

        input_noanom = XSDataMinimalXdsIn()
        input_noanom.input_file = XSDataString(new_xdsinp)
        input_noanom.friedels_law = XSDataBoolean(True)
        input_noanom.job = XSDataString('CORRECT')
        input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_noanom.dataInput = input_noanom

        xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
        xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())


        # let's make some links!
        for f in self._to_link:
            os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
            os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
示例#12
0
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG('EDPluginControlAutoproc.preProcess starting')
        self.DEBUG('failure state is currently {0}'.format(self.isFailure()))

        # for info to send to the autoproc stats server
        self.custom_stats = dict(creation_time=time.time(),
                                 processing_type='edna fastproc',
                                 datacollect_id=self.dataInput.data_collection_id.value,
                                 comments='running on {0}'.format(socket.gethostname()))


        data_in = self.dataInput
        xds_in = XSDataMinimalXdsIn()
        xds_in.input_file = data_in.input_file.path
        xds_in.spacegroup = data_in.spacegroup
        xds_in.unit_cell = data_in.unit_cell

        self.log_file_path = os.path.join(self.root_dir, 'stats.json')
        self.DEBUG('will log timing information to {0}'.format(self.log_file_path))
        self.stats = dict()

        # Get the image prefix from the directory name
        # XXX: This is horrible
        try:
            self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1])
        except Exception:
            self.image_prefix = ''

        self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing')
        try:
            os.makedirs(self.results_dir)
        except OSError: # it most likely exists
            pass

        # Copy the vanilla XDS input file to the results dir
        infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP')
        shutil.copy(self.dataInput.input_file.path.value,
                    infile_dest)

        # Ensure the autoproc ids directory is there
        self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids')
        try:
            os.makedirs(self.autoproc_ids_dir)
        except OSError: # it's there
            pass


        # we'll need the low res limit later on
        lowres = data_in.low_resolution_limit
        if lowres is not None:
            self.low_resolution_limit = lowres.value
        else:
            self.low_resolution_limit = 50

        res_override = data_in.res_override
        if res_override is not None:
            self.res_override = res_override.value
        else:
            # XXX: default to 0?
            self.res_override = None

        # check the number of images (must be > 8) and get the first
        # image name to wait for. Also modify the XDS.INP file to
        # reflect these values, if specified
        conf = parse_xds_file(data_in.input_file.path.value)


        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in conf:
            xgeo = os.path.abspath(os.path.join(self.root_dir,
                                                conf['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo))
                del conf['X-GEO_CORR=']
            else:
                conf['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in conf:
            ygeo = os.path.abspath(os.path.join(self.root_dir,
                                                conf['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo))
                del conf['Y-GEO_CORR=']
            else:
                conf['Y-GEO_CORR='] = ygeo

        dump_xds_file(data_in.input_file.path.value, conf)

        resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')

        if resrange is not None:
            if self.low_resolution_limit is not None:
                resrange[0] = self.low_resolution_limit
            if self.res_override is not None:
                resrange[1] = self.res_override
            conf['INCLUDE_RESOLUTION_RANGE='] = resrange
            dump_xds_file(data_in.input_file.path.value, conf)


        data_range = conf.get('DATA_RANGE=')
        # we'll need that for the very last part ( file import )
        self.data_range = data_range
        if data_range is not None:
            start_image = data_range[0]
            end_image = data_range[1]
            if end_image - start_image < 8:
                self.ERROR('there are fewer than 8 images, aborting')
                self.setFailure()
                return

        template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        self.DEBUG('template for images is {0}'.format(template))
        # fix the path if it's not absolute
        if not os.path.isabs(template):
            self.DEBUG('file template {0} is not absolute'.format(template))
            base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value))
            template = os.path.normpath(os.path.join(self.root_dir, template))
            conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template
            self.DEBUG('file template fixed to {0}'.format(template))
            self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value))
            dump_xds_file(data_in.input_file.path.value, conf)

        first_image = _template_to_image(template, start_image)


        self.wait_file = self.loadPlugin('EDPluginWaitFile')
        waitfileinput = XSDataInputWaitFile()
        waitfileinput.expectedFile = XSDataFile()
        waitfileinput.expectedFile.path = XSDataString(first_image)
        waitfileinput.expectedSize = XSDataInteger(0) # we do not care
        timeout = XSDataTime()
        global WAIT_FOR_FRAME_TIMEOUT
        timeout.value = WAIT_FOR_FRAME_TIMEOUT
        waitfileinput.timeOut = timeout
        self.wait_file.dataInput = waitfileinput

        self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
        self.xds_first.dataInput = xds_in

        self.generate = self.loadPlugin("EDPluginXDSGenerate")

        self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")

        self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
        self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")

        self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")

        self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')
        self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4')

        self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')

        self.DEBUG('EDPluginControlAutoproc.preProcess finished')
示例#13
0
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAutoproc.preProcess")

        self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
        self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')

        path = os.path.abspath(self.dataInput.previous_run_dir.value)

        # create the data inputs now we know the files are here
        input_anom = XSDataMinimalXdsIn()
        input_anom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
        input_anom.friedels_law = XSDataBoolean(True)
        input_anom.job = XSDataString('CORRECT')
        input_anom.resolution = self.dataInput.resolution
        input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_anom.dataInput = input_anom

        input_noanom = XSDataMinimalXdsIn()
        input_noanom.input_file = XSDataString(os.path.join(path, 'XDS.INP'))
        input_noanom.fridels_law = XSDataBoolean(False)
        input_noanom.job = XSDataString('CORRECT')
        input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_noanom.dataInput = input_noanom

        xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
        xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())
        # let's make some links!
        for f in self._to_link:
            os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
            os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))
        # now this is the horrible part, we need to make symlinks to
        # the images also. for now we rely on the fact that the links
        # in the previous run are most likely the links to the
        # images. So we will make the same links and rely on the fact
        # that in the input file the path to the images is already
        # relative to the CWD
        path = os.path.abspath(self.dataInput.previous_run_dir.value)
        for f in os.listdir(path):
            fullpath = os.path.join(path, f)
            if os.path.islink(fullpath):
                # symlink the symlink...
                os.symlink(fullpath, os.path.join(xds_anom_dir, f))
                os.symlink(fullpath, os.path.join(xds_noanom_dir, f))
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlXdsBest.preProcess")
        self._xds = self.loadPlugin('EDPluginExecMinimalXds')

        # save the root path (where the initial xds.inp is) for later use
        self.root_dir = os.path.abspath(
            os.path.dirname(self.dataInput.input_file.value))

        # let's begin by copying the input file to avoid clobbering it
        shutil.copy(self.dataInput.input_file.value,
                    self.getWorkingDirectory())
        self.real_input_file = os.path.join(
            self.getWorkingDirectory(),
            os.path.basename(self.dataInput.input_file.value))

        # update the keywords
        self.parsed_config = parse_xds_file(self.real_input_file)
        di = self.dataInput
        unit_cell_constants = '{0:.2f} {1:.2f} {2:.2f} {3:.2f} {4:.2f} {5:.2f}'.format(
            di.unit_cell_a.value, di.unit_cell_b.value, di.unit_cell_c.value,
            di.unit_cell_alpha.value, di.unit_cell_beta.value,
            di.unit_cell_gamma.value)
        sg = int(di.space_group.value)
        self.parsed_config['UNIT_CELL_CONSTANTS='] = unit_cell_constants
        self.parsed_config['SPACE_GROUP_NUMBER='] = sg

        # to avoid any problem let's also make the image template
        # absolute as well. The underlying XDS template will take care
        # of creating the links in its dir
        imtemplate = self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        basedir = os.path.abspath(
            os.path.dirname(self.dataInput.input_file.value))
        newpath = os.path.join(basedir, imtemplate)
        self.parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = newpath
        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in self.parsed_config:
            xgeo = os.path.abspath(
                os.path.join(self.root_dir,
                             self.parsed_config['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(xgeo))
                del self.parsed_config['X-GEO_CORR=']
            else:
                self.parsed_config['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in self.parsed_config:
            ygeo = os.path.abspath(
                os.path.join(self.root_dir,
                             self.parsed_config['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(ygeo))
                del self.parsed_config['Y-GEO_CORR=']
            else:
                self.parsed_config['Y-GEO_CORR='] = ygeo
        dump_xds_file(self.real_input_file, self.parsed_config)

        # create the input data model for the XDS plugin
        input_dm = XSDataMinimalXdsIn()
        input_dm.input_file = XSDataString(self.real_input_file)
        self._xds.dataInput = input_dm
示例#15
0
    def preProcess(self, _edObject=None):
        EDPluginControl.preProcess(self)
        self.DEBUG('EDPluginControlAutoproc.preProcess starting')
        self.DEBUG('failure state is currently {0}'.format(self.isFailure()))

        # for info to send to the autoproc stats server
        self.custom_stats = dict(
            creation_time=time.time(),
            processing_type='edna fastproc',
            datacollect_id=self.dataInput.data_collection_id.value,
            comments='running on {0}'.format(socket.gethostname()))

        data_in = self.dataInput
        xds_in = XSDataMinimalXdsIn()
        xds_in.input_file = data_in.input_file.path
        xds_in.spacegroup = data_in.spacegroup
        xds_in.unit_cell = data_in.unit_cell

        self.log_file_path = os.path.join(self.root_dir, 'stats.json')
        self.DEBUG('will log timing information to {0}'.format(
            self.log_file_path))
        self.stats = dict()

        # Get the image prefix from the directory name
        # XXX: This is horrible
        try:
            self.image_prefix = '_'.join(
                os.path.basename(self.root_dir).split('_')[1:-1])
        except Exception:
            self.image_prefix = ''

        self.results_dir = os.path.join(self.root_dir, 'results',
                                        'fast_processing')
        try:
            os.makedirs(self.results_dir)
        except OSError:  # it most likely exists
            pass

        # Copy the vanilla XDS input file to the results dir
        infile_dest = os.path.join(self.results_dir,
                                   self.image_prefix + '_input_XDS.INP')
        shutil.copy(self.dataInput.input_file.path.value, infile_dest)

        # Ensure the autoproc ids directory is there
        self.autoproc_ids_dir = os.path.join(self.results_dir,
                                             'fastproc_integration_ids')
        try:
            os.makedirs(self.autoproc_ids_dir)
        except OSError:  # it's there
            pass

        # we'll need the low res limit later on
        lowres = data_in.low_resolution_limit
        if lowres is not None:
            self.low_resolution_limit = lowres.value
        else:
            self.low_resolution_limit = 50

        res_override = data_in.res_override
        if res_override is not None:
            self.res_override = res_override.value
        else:
            # XXX: default to 0?
            self.res_override = None

        # check the number of images (must be > 8) and get the first
        # image name to wait for. Also modify the XDS.INP file to
        # reflect these values, if specified
        conf = parse_xds_file(data_in.input_file.path.value)

        # Make the [XY]-GEO_CORR paths absolute
        if 'X-GEO_CORR=' in conf:
            xgeo = os.path.abspath(
                os.path.join(self.root_dir, conf['X-GEO_CORR='][0]))
            if not os.path.exists(xgeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(xgeo))
                del conf['X-GEO_CORR=']
            else:
                conf['X-GEO_CORR='] = xgeo

        if 'Y-GEO_CORR=' in conf:
            ygeo = os.path.abspath(
                os.path.join(self.root_dir, conf['Y-GEO_CORR='][0]))
            if not os.path.exists(ygeo):
                self.DEBUG(
                    'geometry file {0} does not exist, removing'.format(ygeo))
                del conf['Y-GEO_CORR=']
            else:
                conf['Y-GEO_CORR='] = ygeo

        dump_xds_file(data_in.input_file.path.value, conf)

        resrange = conf.get('INCLUDE_RESOLUTION_RANGE=')

        if resrange is not None:
            if self.low_resolution_limit is not None:
                resrange[0] = self.low_resolution_limit
            if self.res_override is not None:
                resrange[1] = self.res_override
            conf['INCLUDE_RESOLUTION_RANGE='] = resrange
            dump_xds_file(data_in.input_file.path.value, conf)

        data_range = conf.get('DATA_RANGE=')
        # we'll need that for the very last part ( file import )
        self.data_range = data_range
        if data_range is not None:
            start_image = data_range[0]
            end_image = data_range[1]
            if end_image - start_image < 8:
                self.ERROR('there are fewer than 8 images, aborting')
                self.setFailure()
                return

        template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        self.DEBUG('template for images is {0}'.format(template))
        # fix the path if it's not absolute
        if not os.path.isabs(template):
            self.DEBUG('file template {0} is not absolute'.format(template))
            base_dir = os.path.abspath(
                os.path.dirname(data_in.input_file.path.value))
            template = os.path.normpath(os.path.join(self.root_dir, template))
            conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template
            self.DEBUG('file template fixed to {0}'.format(template))
            self.DEBUG('dumping back the file to {0}'.format(
                data_in.input_file.path.value))
            dump_xds_file(data_in.input_file.path.value, conf)

        first_image = _template_to_image(template, start_image)

        self.wait_file = self.loadPlugin('EDPluginWaitFile')
        waitfileinput = XSDataInputWaitFile()
        waitfileinput.expectedFile = XSDataFile()
        waitfileinput.expectedFile.path = XSDataString(first_image)
        waitfileinput.expectedSize = XSDataInteger(0)  # we do not care
        timeout = XSDataTime()
        global WAIT_FOR_FRAME_TIMEOUT
        timeout.value = WAIT_FOR_FRAME_TIMEOUT
        waitfileinput.timeOut = timeout
        self.wait_file.dataInput = waitfileinput

        self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc")
        self.xds_first.dataInput = xds_in

        self.generate = self.loadPlugin("EDPluginXDSGenerate")

        self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff")
        self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff")

        self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput")
        self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput")

        self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate")

        self.store_autoproc_anom = self.loadPlugin(
            'EDPluginISPyBStoreAutoProcv1_4')
        self.store_autoproc_noanom = self.loadPlugin(
            'EDPluginISPyBStoreAutoProcv1_4')

        self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport')

        self.DEBUG('EDPluginControlAutoproc.preProcess finished')
示例#16
0
    def preProcess(self, _edObject = None):
        EDPluginControl.preProcess(self)
        self.DEBUG("EDPluginControlAutoproc.preProcess")

        self.xds_anom = self.loadPlugin('EDPluginExecMinimalXds')
        self.xds_noanom = self.loadPlugin('EDPluginExecMinimalXds')

        path = os.path.abspath(self.dataInput.previous_run_dir.value)

        # The MinimalXds plugin takes care of creating determining the
        # real images directory and creating a symlink to it so we
        # only need to update the NAMED_TEMPLATE_OF_DATA_FILES keyword
        # to not be relative anymore. We'll copy it to our own dir
        # beforehand to avoid clobbering it
        xdsinp = os.path.join(path, 'XDS.INP')
        new_xdsinp = os.path.join(self.getWorkingDirectory(), 'XDS.INP')
        copyfile(xdsinp, new_xdsinp)


        parsed_config = parse_xds_file(new_xdsinp)
        file_template = parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='][0]
        parsed_config['NAME_TEMPLATE_OF_DATA_FRAMES='] = os.path.abspath(os.path.join(path, file_template))
        dump_xds_file(new_xdsinp, parsed_config)

        # create the data inputs now we know the files are here
        input_anom = XSDataMinimalXdsIn()
        input_anom.input_file = XSDataString(new_xdsinp)
        input_anom.friedels_law = XSDataBoolean(True)
        input_anom.job = XSDataString('CORRECT')
        input_anom.resolution = self.dataInput.resolution
        input_anom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_anom.dataInput = input_anom

        input_noanom = XSDataMinimalXdsIn()
        input_noanom.input_file = XSDataString(new_xdsinp)
        input_noanom.friedels_law = XSDataBoolean(False)
        input_noanom.job = XSDataString('CORRECT')
        input_noanom.resolution_range = [XSDataFloat(60), self.dataInput.resolution]
        self.xds_noanom.dataInput = input_noanom

        xds_anom_dir = os.path.abspath(self.xds_anom.getWorkingDirectory())
        xds_noanom_dir = os.path.abspath(self.xds_noanom.getWorkingDirectory())


        # let's make some links!
        for f in self._to_link:
            os.symlink(f, os.path.join(xds_anom_dir, os.path.basename(f)))
            os.symlink(f, os.path.join(xds_noanom_dir, os.path.basename(f)))