def process(self, _edObject = None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunXdsFastProc.process") # First run is vanilla without any modification params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell self.first_run.dataInput = params self.first_run.executeSynchronous() EDVerbose.DEBUG('first run completed...') if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.first_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.second_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG('copying previously generated files to the new plugin dir') copy_xds_files(self.first_run.getWorkingDirectory(), self.second_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell # increase all the spot ranges end by 20, constrained to # the data range upper limit spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 20 if range_end > self.end_image_no: self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.second_run.dataInput = params self.second_run.executeSynchronous() EDVerbose.DEBUG('second run completed') if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.second_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.third_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG('copying previously generated files to the new plugin dir') copy_xds_files(self.second_run.getWorkingDirectory(), self.third_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 40 if range_end > self.end_image_no: self.DEBUG('End of range {0} would be past the last image {1}'.format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format(srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.third_run.dataInput = params self.third_run.executeSynchronous() EDVerbose.DEBUG('third run completed') if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.third_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # all runs failed so bail out ... self.setFailure() else: # use the xds parser plugin to parse the xds output file... parser = self.loadPlugin("EDPluginParseXdsOutput") wd = self.successful_run.getWorkingDirectory() parser_input = XSDataXdsOutputFile() correct_lp_path = XSDataFile() correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP')) parser_input.correct_lp = correct_lp_path gxparm_path = os.path.join(wd, 'GXPARM.XDS') if os.path.isfile(gxparm_path): gxparm = XSDataFile() gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS')) parser_input.gxparm = gxparm parser.dataInput = parser_input parser.executeSynchronous() if parser.isFailure(): # that should not happen self.setFailure() return self.dataOutput = parser.dataOutput
def preProcess(self, _edObject = None): EDPluginControl.preProcess(self) self.DEBUG('EDPluginControlAutoproc.preProcess starting') self.DEBUG('failure state is currently {0}'.format(self.isFailure())) # for info to send to the autoproc stats server self.custom_stats = dict(creation_time=time.time(), processing_type='edna fastproc', datacollect_id=self.dataInput.data_collection_id.value, comments='running on {0}'.format(socket.gethostname())) data_in = self.dataInput xds_in = XSDataMinimalXdsIn() xds_in.input_file = data_in.input_file.path xds_in.spacegroup = data_in.spacegroup xds_in.unit_cell = data_in.unit_cell self.log_file_path = os.path.join(self.root_dir, 'stats.json') self.DEBUG('will log timing information to {0}'.format(self.log_file_path)) self.stats = dict() # Get the image prefix from the directory name # XXX: This is horrible try: self.image_prefix = '_'.join(os.path.basename(self.root_dir).split('_')[1:-1]) except Exception: self.image_prefix = '' self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing') try: os.makedirs(self.results_dir) except OSError: # it most likely exists pass # Copy the vanilla XDS input file to the results dir infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP') shutil.copy(self.dataInput.input_file.path.value, infile_dest) # Ensure the autoproc ids directory is there self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids') try: os.makedirs(self.autoproc_ids_dir) except OSError: # it's there pass # we'll need the low res limit later on lowres = data_in.low_resolution_limit if lowres is not None: self.low_resolution_limit = lowres.value else: self.low_resolution_limit = 50 res_override = data_in.res_override if res_override is not None: self.res_override = res_override.value else: # XXX: default to 0? self.res_override = None # check the number of images (must be > 8) and get the first # image name to wait for. Also modify the XDS.INP file to # reflect these values, if specified conf = parse_xds_file(data_in.input_file.path.value) # Make the [XY]-GEO_CORR paths absolute if 'X-GEO_CORR=' in conf: xgeo = os.path.abspath(os.path.join(self.root_dir, conf['X-GEO_CORR='][0])) if not os.path.exists(xgeo): self.DEBUG('geometry file {0} does not exist, removing'.format(xgeo)) del conf['X-GEO_CORR='] else: conf['X-GEO_CORR='] = xgeo if 'Y-GEO_CORR=' in conf: ygeo = os.path.abspath(os.path.join(self.root_dir, conf['Y-GEO_CORR='][0])) if not os.path.exists(ygeo): self.DEBUG('geometry file {0} does not exist, removing'.format(ygeo)) del conf['Y-GEO_CORR='] else: conf['Y-GEO_CORR='] = ygeo dump_xds_file(data_in.input_file.path.value, conf) resrange = conf.get('INCLUDE_RESOLUTION_RANGE=') if resrange is not None: if self.low_resolution_limit is not None: resrange[0] = self.low_resolution_limit if self.res_override is not None: resrange[1] = self.res_override conf['INCLUDE_RESOLUTION_RANGE='] = resrange dump_xds_file(data_in.input_file.path.value, conf) data_range = conf.get('DATA_RANGE=') # we'll need that for the very last part ( file import ) self.data_range = data_range if data_range is not None: start_image = data_range[0] end_image = data_range[1] if end_image - start_image < 8: self.ERROR('there are fewer than 8 images, aborting') self.setFailure() return template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0] self.DEBUG('template for images is {0}'.format(template)) # fix the path if it's not absolute if not os.path.isabs(template): self.DEBUG('file template {0} is not absolute'.format(template)) base_dir = os.path.abspath(os.path.dirname(data_in.input_file.path.value)) template = os.path.normpath(os.path.join(self.root_dir, template)) conf['NAME_TEMPLATE_OF_DATA_FRAMES=']=template self.DEBUG('file template fixed to {0}'.format(template)) self.DEBUG('dumping back the file to {0}'.format(data_in.input_file.path.value)) dump_xds_file(data_in.input_file.path.value, conf) first_image = _template_to_image(template, start_image) self.wait_file = self.loadPlugin('EDPluginWaitFile') waitfileinput = XSDataInputWaitFile() waitfileinput.expectedFile = XSDataFile() waitfileinput.expectedFile.path = XSDataString(first_image) waitfileinput.expectedSize = XSDataInteger(0) # we do not care timeout = XSDataTime() global WAIT_FOR_FRAME_TIMEOUT timeout.value = WAIT_FOR_FRAME_TIMEOUT waitfileinput.timeOut = timeout self.wait_file.dataInput = waitfileinput self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc") self.xds_first.dataInput = xds_in self.generate = self.loadPlugin("EDPluginXDSGenerate") self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff") self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput") self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput") self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate") self.store_autoproc_anom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4') self.store_autoproc_noanom = self.loadPlugin('EDPluginISPyBStoreAutoProcv1_4') self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport') self.DEBUG('EDPluginControlAutoproc.preProcess finished')
def process(self, _edObject=None): EDPluginControl.process(self) self.DEBUG("EDPluginControlRunXdsFastProc.process") # First run is vanilla without any modification params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell self.first_run.dataInput = params self.first_run.executeSynchronous() EDVerbose.DEBUG('first run completed...') if self.first_run.dataOutput is not None and self.first_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.first_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.second_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG( 'copying previously generated files to the new plugin dir') copy_xds_files(self.first_run.getWorkingDirectory(), self.second_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell # increase all the spot ranges end by 20, constrained to # the data range upper limit spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 20 if range_end > self.end_image_no: self.DEBUG( 'End of range {0} would be past the last image {1}'. format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format( srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.second_run.dataInput = params self.second_run.executeSynchronous() EDVerbose.DEBUG('second run completed') if self.second_run.dataOutput is not None and self.second_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.second_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: self.third_run = self.loadPlugin(self.controlled_plugin_name) self.DEBUG('retrying with increased SPOT_RANGE') self.DEBUG( 'copying previously generated files to the new plugin dir') copy_xds_files(self.second_run.getWorkingDirectory(), self.third_run.getWorkingDirectory()) params = XSDataMinimalXdsIn() params.input_file = self.dataInput.input_file params.job = XSDataString('DEFPIX INTEGRATE CORRECT') params.spacegroup = self.dataInput.spacegroup params.unit_cell = self.dataInput.unit_cell spot_range = list() for srange in self.spot_range: range_begin = srange[0] range_end = srange[1] + 40 if range_end > self.end_image_no: self.DEBUG( 'End of range {0} would be past the last image {1}'. format(range_end, self.end_image_no)) range_end = self.end_image_no self.DEBUG('Changing spot range {0} to [{1} {2}]'.format( srange, range_begin, range_end)) r = XSDataRange(begin=range_begin, end=range_end) spot_range.append(r) params.spot_range = spot_range self.third_run.dataInput = params self.third_run.executeSynchronous() EDVerbose.DEBUG('third run completed') if self.third_run.dataOutput is not None and self.third_run.dataOutput.succeeded.value: EDVerbose.DEBUG('... and it worked') self.successful_run = self.third_run else: EDVerbose.DEBUG('... and it failed') if not self.successful_run: # all runs failed so bail out ... self.setFailure() else: # use the xds parser plugin to parse the xds output file... parser = self.loadPlugin("EDPluginParseXdsOutput") wd = self.successful_run.getWorkingDirectory() parser_input = XSDataXdsOutputFile() correct_lp_path = XSDataFile() correct_lp_path.path = XSDataString(os.path.join(wd, 'CORRECT.LP')) parser_input.correct_lp = correct_lp_path gxparm_path = os.path.join(wd, 'GXPARM.XDS') if os.path.isfile(gxparm_path): gxparm = XSDataFile() gxparm.path = XSDataString(os.path.join(wd, 'GXPARM.XDS')) parser_input.gxparm = gxparm parser.dataInput = parser_input parser.executeSynchronous() if parser.isFailure(): # that should not happen self.setFailure() return self.dataOutput = parser.dataOutput
def preProcess(self, _edObject=None): EDPluginControl.preProcess(self) self.DEBUG('EDPluginControlAutoproc.preProcess starting') self.DEBUG('failure state is currently {0}'.format(self.isFailure())) # for info to send to the autoproc stats server self.custom_stats = dict( creation_time=time.time(), processing_type='edna fastproc', datacollect_id=self.dataInput.data_collection_id.value, comments='running on {0}'.format(socket.gethostname())) data_in = self.dataInput xds_in = XSDataMinimalXdsIn() xds_in.input_file = data_in.input_file.path xds_in.spacegroup = data_in.spacegroup xds_in.unit_cell = data_in.unit_cell self.log_file_path = os.path.join(self.root_dir, 'stats.json') self.DEBUG('will log timing information to {0}'.format( self.log_file_path)) self.stats = dict() # Get the image prefix from the directory name # XXX: This is horrible try: self.image_prefix = '_'.join( os.path.basename(self.root_dir).split('_')[1:-1]) except Exception: self.image_prefix = '' self.results_dir = os.path.join(self.root_dir, 'results', 'fast_processing') try: os.makedirs(self.results_dir) except OSError: # it most likely exists pass # Copy the vanilla XDS input file to the results dir infile_dest = os.path.join(self.results_dir, self.image_prefix + '_input_XDS.INP') shutil.copy(self.dataInput.input_file.path.value, infile_dest) # Ensure the autoproc ids directory is there self.autoproc_ids_dir = os.path.join(self.results_dir, 'fastproc_integration_ids') try: os.makedirs(self.autoproc_ids_dir) except OSError: # it's there pass # we'll need the low res limit later on lowres = data_in.low_resolution_limit if lowres is not None: self.low_resolution_limit = lowres.value else: self.low_resolution_limit = 50 res_override = data_in.res_override if res_override is not None: self.res_override = res_override.value else: # XXX: default to 0? self.res_override = None # check the number of images (must be > 8) and get the first # image name to wait for. Also modify the XDS.INP file to # reflect these values, if specified conf = parse_xds_file(data_in.input_file.path.value) # Make the [XY]-GEO_CORR paths absolute if 'X-GEO_CORR=' in conf: xgeo = os.path.abspath( os.path.join(self.root_dir, conf['X-GEO_CORR='][0])) if not os.path.exists(xgeo): self.DEBUG( 'geometry file {0} does not exist, removing'.format(xgeo)) del conf['X-GEO_CORR='] else: conf['X-GEO_CORR='] = xgeo if 'Y-GEO_CORR=' in conf: ygeo = os.path.abspath( os.path.join(self.root_dir, conf['Y-GEO_CORR='][0])) if not os.path.exists(ygeo): self.DEBUG( 'geometry file {0} does not exist, removing'.format(ygeo)) del conf['Y-GEO_CORR='] else: conf['Y-GEO_CORR='] = ygeo dump_xds_file(data_in.input_file.path.value, conf) resrange = conf.get('INCLUDE_RESOLUTION_RANGE=') if resrange is not None: if self.low_resolution_limit is not None: resrange[0] = self.low_resolution_limit if self.res_override is not None: resrange[1] = self.res_override conf['INCLUDE_RESOLUTION_RANGE='] = resrange dump_xds_file(data_in.input_file.path.value, conf) data_range = conf.get('DATA_RANGE=') # we'll need that for the very last part ( file import ) self.data_range = data_range if data_range is not None: start_image = data_range[0] end_image = data_range[1] if end_image - start_image < 8: self.ERROR('there are fewer than 8 images, aborting') self.setFailure() return template = conf['NAME_TEMPLATE_OF_DATA_FRAMES='][0] self.DEBUG('template for images is {0}'.format(template)) # fix the path if it's not absolute if not os.path.isabs(template): self.DEBUG('file template {0} is not absolute'.format(template)) base_dir = os.path.abspath( os.path.dirname(data_in.input_file.path.value)) template = os.path.normpath(os.path.join(self.root_dir, template)) conf['NAME_TEMPLATE_OF_DATA_FRAMES='] = template self.DEBUG('file template fixed to {0}'.format(template)) self.DEBUG('dumping back the file to {0}'.format( data_in.input_file.path.value)) dump_xds_file(data_in.input_file.path.value, conf) first_image = _template_to_image(template, start_image) self.wait_file = self.loadPlugin('EDPluginWaitFile') waitfileinput = XSDataInputWaitFile() waitfileinput.expectedFile = XSDataFile() waitfileinput.expectedFile.path = XSDataString(first_image) waitfileinput.expectedSize = XSDataInteger(0) # we do not care timeout = XSDataTime() global WAIT_FOR_FRAME_TIMEOUT timeout.value = WAIT_FOR_FRAME_TIMEOUT waitfileinput.timeOut = timeout self.wait_file.dataInput = waitfileinput self.xds_first = self.loadPlugin("EDPluginControlRunXdsFastProc") self.xds_first.dataInput = xds_in self.generate = self.loadPlugin("EDPluginXDSGenerate") self.first_res_cutoff = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_anom = self.loadPlugin("EDPluginResCutoff") self.res_cutoff_noanom = self.loadPlugin("EDPluginResCutoff") self.parse_xds_anom = self.loadPlugin("EDPluginParseXdsOutput") self.parse_xds_noanom = self.loadPlugin("EDPluginParseXdsOutput") self.xscale_generate = self.loadPlugin("EDPluginControlXscaleGenerate") self.store_autoproc_anom = self.loadPlugin( 'EDPluginISPyBStoreAutoProcv1_4') self.store_autoproc_noanom = self.loadPlugin( 'EDPluginISPyBStoreAutoProcv1_4') self.file_conversion = self.loadPlugin('EDPluginControlAutoprocImport') self.DEBUG('EDPluginControlAutoproc.preProcess finished')