def process(self, _edObject = None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunXdsFastProc.process") # First run is vanilla without any modification params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell self.first_run.dataInput = params self.first_run.executeSynchronous() EDVerbose.DEBUG('first run completed...') if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.first_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.second_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG('copying previously generated files to the new plugin dir') copy_xds_files(self.first_run.getWorkingDirectory(), self.second_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell # increase all the spot ranges end by 20, constrained to # the data range upper limit spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 20 if range_end > self.end_image_no: self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.second_run.dataInput = params self.second_run.executeSynchronous() EDVerbose.DEBUG('second run completed') if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.second_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.third_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG('copying previously generated files to the new plugin dir') copy_xds_files(self.second_run.getWorkingDirectory(), self.third_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 40 if range_end > self.end_image_no: self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.third_run.dataInput = params self.third_run.executeSynchronous() EDVerbose.DEBUG('third run completed') if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.third_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # all runs failed so bail out ... self.setFailure() else: # use the xds parser plugin to parse the xds output file... parser = self.loadPlugin("EDPluginParseXdsOutput") wd = self.successful_run.getWorkingDirectory() parser_input = XSDataXdsOutputFile() correct_lp_path = XSDataFile() correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP')) parser_input.correct_lp = correct_lp_path gxparm_path = os.path.join(wd, 'GXPARM.XDS') if os.path.isfile(gxparm_path): gxparm = XSDataFile() gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS')) parser_input.gxparm = gxparm parser.dataInput = parser_input parser.executeSynchronous() if parser.isFailure(): # that should not happen self.setFailure() return self.dataOutput = parser.dataOutput
def process(self, _edObject = None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunXds.process") # First run is vanilla without any modification params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file self.first_run.dataInput = params self.first_run.executeSynchronous() EDVerbose.DEBUG('first run completed...') if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.first_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # second run w/ JOB set to DEFPIX INTEGRATE CORRECT self.second_run = self.loadPlugin(self.controlled_plugin_name) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.jobs = 'DEFPIX INTEGRATE CORRECT' self.second_run.dataInput = params self.second_run.executeSynchronous() EDVerbose.DEBUG('second run completed') if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.second_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1 self.third_run = self.loadPlugin(self.controlled_plugin_name) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.jobs = 'ALL' params.maxprocs = 4 params.maxjobs = 1 self.third_run.dataInput = params self.third_run.executeSynchronous() EDVerbose.DEBUG('third run completed') if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.third_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # final run with parallelism like 3 but JOB like 2 self.final_run = self.loadPlugin(self.controlled_plugin_name) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.jobs = 'DEFPIX INTEGRATE CORRECT' params.maxprocs = 4 params.maxjobs = 1 self.final_run.dataInput = params self.final_run.executeSynchronous() EDVerbose.DEBUG('final run completed') if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded: EDVerbose.DEBUG('... and it worked') self.successful_run = self.final_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # all runs failed so bail out ... self.setFailure() else: # use the xds parser plugin to parse the xds output file... parser = self.loadPlugin("EDPluginParseXdsOutput") wd = self.successful_run.getWorkingDirectory() parser_input = XSDataXdsOutputFile() correct_lp_path = XSDataFile() correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP')) parser_input.correct_lp = correct_lp_path gxparm_path = os.path.join(wd, 'GXPARM.XDS') if os.path.isfile(gxparm_path): gxparm = XSDataFile() gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS')) parser_input.gxparm = gxparm parser.dataInput = parser_input parser.executeSynchronous() if parser.isFailure(): # that should not happen self.setFailure() return self.dataOutput = parser.dataOutput
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunXdsFastProc.process") # First run is vanilla without any modification params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell self.first_run.dataInput = params self.first_run.executeSynchronous() EDVerbose.DEBUG('first run completed...') if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.first_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.second_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG( 'copying previously generated files to the new plugin dir') copy_xds_files(self.first_run.getWorkingDirectory(), self.second_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell # increase all the spot ranges end by 20, constrained to # the data range upper limit spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 20 if range_end > self.end_image_no: self.DEBUG( 'End of range {0} would be past the last image {1}'. format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format( srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.second_run.dataInput = params self.second_run.executeSynchronous() EDVerbose.DEBUG('second run completed') if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.second_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.third_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG( 'copying previously generated files to the new plugin dir') copy_xds_files(self.second_run.getWorkingDirectory(), self.third_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 40 if range_end > self.end_image_no: self.DEBUG( 'End of range {0} would be past the last image {1}'. format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format( srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.third_run.dataInput = params self.third_run.executeSynchronous() EDVerbose.DEBUG('third run completed') if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.third_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # all runs failed so bail out ... self.setFailure() else: # use the xds parser plugin to parse the xds output file... parser = self.loadPlugin("EDPluginParseXdsOutput") wd = self.successful_run.getWorkingDirectory() parser_input = XSDataXdsOutputFile() correct_lp_path = XSDataFile() correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP')) parser_input.correct_lp = correct_lp_path gxparm_path = os.path.join(wd, 'GXPARM.XDS') if os.path.isfile(gxparm_path): gxparm = XSDataFile() gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS')) parser_input.gxparm = gxparm parser.dataInput = parser_input parser.executeSynchronous() if parser.isFailure(): # that should not happen self.setFailure() return self.dataOutput = parser.dataOutput
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunXds.process") # First run is vanilla without any modification params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file self.first_run.dataInput = params self.first_run.executeSynchronous() EDVerbose.DEBUG('first run completed...') if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded: EDVerbose.DEBUG('... and it worked') self.successful_run = self.first_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # second run w/ JOB set to DEFPIX INTEGRATE CORRECT self.second_run = self.loadPlugin(self.controlled_plugin_name) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.jobs = 'DEFPIX INTEGRATE CORRECT' self.second_run.dataInput = params self.second_run.executeSynchronous() EDVerbose.DEBUG('second run completed') if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded: EDVerbose.DEBUG('... and it worked') self.successful_run = self.second_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # third run with JOB set to ALL and mxaprocs = 4 and maxjobs = 1 self.third_run = self.loadPlugin(self.controlled_plugin_name) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.jobs = 'ALL' params.maxprocs = 4 params.maxjobs = 1 self.third_run.dataInput = params self.third_run.executeSynchronous() EDVerbose.DEBUG('third run completed') if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded: EDVerbose.DEBUG('... and it worked') self.successful_run = self.third_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # final run with parallelism like 3 but JOB like 2 self.final_run = self.loadPlugin(self.controlled_plugin_name) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.jobs = 'DEFPIX INTEGRATE CORRECT' params.maxprocs = 4 params.maxjobs = 1 self.final_run.dataInput = params self.final_run.executeSynchronous() EDVerbose.DEBUG('final run completed') if self.final_run.dataOutput is not None and self.final_run.dataOutput.succeeded: EDVerbose.DEBUG('... and it worked') self.successful_run = self.final_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # all runs failed so bail out ... self.setFailure() else: # use the xds parser plugin to parse the xds output file... parser = self.loadPlugin("EDPluginParseXdsOutput") wd = self.successful_run.getWorkingDirectory() parser_input = XSDataXdsOutputFile() correct_lp_path = XSDataFile() correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP')) parser_input.correct_lp = correct_lp_path gxparm_path = os.path.join(wd, 'GXPARM.XDS') if os.path.isfile(gxparm_path): gxparm = XSDataFile() gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS')) parser_input.gxparm = gxparm parser.dataInput = parser_input parser.executeSynchronous() if parser.isFailure(): # that should not happen self.setFailure() return self.dataOutput = parser.dataOutput
def process(self, _edObject = None): EDPluginControl.process(self) self.DEBUG("EDPluginControlAutoproc.process") # first XDS plugin run with supplied XDS file EDVerbose.screen("First XDS run...") self.xds_first.executeSynchronous() if self.xds_first.isFailure(): EDVerbose.ERROR('first XDS run failed') self.setFailure() return EDVerbose.screen("First XDS run successfully completed") # apply the first res cutoff with the res extracted from the first XDS run EDVerbose.screen("First resolution cutoff") xdsresult = self.xds_first.dataOutput res_cutoff_in = XSDataResCutoff() res_cutoff_in.xds_res = xdsresult res_cutoff_in.completeness_entries = xdsresult.completeness_entries #XXX: remove from the data model as it is just pass-through? res_cutoff_in.total_completeness = xdsresult.total_completeness self.first_res_cutoff.dataInput = res_cutoff_in self.first_res_cutoff.executeSynchronous() if self.first_res_cutoff.isFailure(): EDVerbose.ERROR("res cutoff failed") self.setFailure() return EDVerbose.screen("First resolution cutoff successfully completed") resolution = self.first_res_cutoff.dataOutput.res # for the generate w/ and w/out anom we have to specify where # the first XDS plugin run took place xds_run_directory = os.path.abspath(self.xds_first.dataOutput.xds_run_directory.value) EDVerbose.screen("the xds run took place in {}".format(xds_run_directory)) generate_input = XSDataXdsGenerateInput() generate_input.resolution = resolution generate_input.previous_run_dir = XSDataString(xds_run_directory) self.generate.dataInput = generate_input self.generate.executeSynchronous() if self.generate.isFailure(): EDVerbose.ERROR('generating w/ and w/out anom failed') self.setFailure() return # we can now use the xds output parser on the two correct.lp # files, w/ and w/out anom parse_anom_input = XSDataXdsOutputFile() parse_anom_input.correct_lp = XSDataFile() parse_anom_input.correct_lp.path = self.generate.dataOutput.correct_lp_anom self.parse_xds_anom.dataInput = parse_anom_input self.parse_xds_anom.executeSynchronous() if self.parse_xds_anom.isFailure(): EDVerbose.ERROR('parsing the xds generated w/ anom failed') self.setFailure() return # now the other one w/out anom parse_noanom_input = XSDataXdsOutputFile() parse_noanom_input.correct_lp = XSDataFile() parse_noanom_input.correct_lp.path = self.generate.dataOutput.correct_lp_no_anom self.parse_xds_noanom.dataInput = parse_noanom_input self.parse_xds_noanom.executeSynchronous() if self.parse_xds_noanom.isFailure(): EDVerbose.ERROR('parsing the xds generated w/ anom failed') self.setFailure() return # we now can apply the res cutoff on the anom and no anom # outputs. Note that this is not done in parallel, like the # xds parsing # XXX completeness_cutoff/res_override and isig_cutoff still # missing res_cutoff_anom_in = XSDataResCutoff() res_cutoff_anom_in.xds_res = self.parse_xds_anom.dataOutput self.res_cutoff_anom.dataInput = res_cutoff_anom_in self.res_cutoff_anom.executeSynchronous() if self.res_cutoff_anom.isFailure(): EDVerbose.ERROR('res cutoff for anom data failed') # same for non anom res_cutoff_noanom_in = XSDataResCutoff() res_cutoff_noanom_in.xds_res = self.parse_xds_noanom.dataOutput self.res_cutoff_noanom.dataInput = res_cutoff_noanom_in self.res_cutoff_noanom.executeSynchronous() if self.res_cutoff_noanom.isFailure(): EDVerbose.ERROR('res cutoff for non anom data failed') # now we just have to run XScale to generate w/ and w/out # anom, merged and unmerged # We use another control plugin for that to isolate the whole thing # if not self.res_cutoff_anom.isFailure(): xscale_anom_in = XSDataXscaleInput() input_file = XSDataXscaleInputFile() input_file.path = self.generate.dataOutput.hkl_anom input_file.res = self.res_cutoff_anom.dataOutput.res xscale_anom_in.xds_files = [input_file] xscale_anom_in.unit_cell_constants = self.parse_xds_anom.dataOutput.unit_cell_constants xscale_anom_in.sg_number = self.parse_xds_anom.dataOutput.sg_number xscale_anom_in.bins = self.res_cutoff_anom.bins self.xscale_anom.dataInput = xscale_anom_in self.xscale_anom.executeSynchronous() if self.xscale_anom.isFailure(): EDVerbose.ERROR('xscale anom/merge generation failed') # same for non anom code path #if not self.res_cutoff_noanom.isFailure(): xscale_noanom_in = XSDataXscaleInput() input_file = XSDataXscaleInputFile() input_file.path = self.generate.dataOuput.hkl_anom input_file.res = self.res_cutoff_anom.dataOutput.res xscale_noanom_in.xds_files = [input_file] xscale_noanom_in.unit_cell_constants = self.parse_xds_noanom.dataOutput.unit_cell_constants xscale_noanom_in.sg_number = self.parse_xds_noanom.dataOutput.sg_number xscale_noanom_in.bins = self.res_cutoff_noanom.bins self.xscale_noanom.dataInput = xscale_noanom_in self.xscale_noanom.executeSynchronous() if self.xscale_noanom.isFailure(): EDVerbose.ERROR('xscale anom/merge generation failed')
self.setFailure() log_to_ispyb([self.integration_id_anom, self.integration_id_noanom], 'Scaling', 'Failed', 'anom/noanom generation failed in {0}s'.format(self.stats['anom/noanom_generation'])) return else: EDVerbose.screen('generating w/ and w/out anom finished') log_to_ispyb([self.integration_id_anom, self.integration_id_noanom], 'Scaling', 'Successful', 'anom/noanom generation finished in {0}s'.format(self.stats['anom/noanom_generation'])) # we can now use the xds output parser on the two correct.lp # files, w/ and w/out anom parse_anom_input = XSDataXdsOutputFile() parse_anom_input.correct_lp = XSDataFile() parse_anom_input.correct_lp.path = self.generate.dataOutput.correct_lp_anom # this one is the same as the first XDS run since they share # the same directory gxparm_file_anom = XSDataFile() gxparm_file_anom.path = self.generate.dataOutput.gxparm parse_anom_input.gxparm = gxparm_file_anom self.parse_xds_anom.dataInput = parse_anom_input self.parse_xds_anom.executeSynchronous() if self.parse_xds_anom.isFailure(): EDVerbose.ERROR('parsing the xds generated w/ anom failed')
[self.integration_id_anom, self.integration_id_noanom], 'Scaling', 'Failed', 'anom/noanom generation failed in {0}s'.format( self.stats['anom/noanom_generation'])) return else: EDVerbose.screen('generating w/ and w/out anom finished') log_to_ispyb( [self.integration_id_anom, self.integration_id_noanom], 'Scaling', 'Successful', 'anom/noanom generation finished in {0}s'.format( self.stats['anom/noanom_generation'])) # we can now use the xds output parser on the two correct.lp # files, w/ and w/out anom parse_anom_input = XSDataXdsOutputFile() parse_anom_input.correct_lp = XSDataFile() parse_anom_input.correct_lp.path = self.generate.dataOutput.correct_lp_anom # this one is the same as the first XDS run since they share # the same directory gxparm_file_anom = XSDataFile() gxparm_file_anom.path = self.generate.dataOutput.gxparm parse_anom_input.gxparm = gxparm_file_anom self.parse_xds_anom.dataInput = parse_anom_input self.parse_xds_anom.executeSynchronous() if self.parse_xds_anom.isFailure(): EDVerbose.ERROR('parsing the xds generated w/ anom failed')