def determine_sg(data_info, dset, options=None): options = options or {} os.chdir(data_info['working_directory']) _logger.info("Automaticaly Determining Symmetry ...") if not misc.file_requirements('INTEGRATE.HKL'): return { 'step': 'symmetry', 'success': False, 'reason': 'Required files from integration missing' } try: programs.pointless(chiral=options.get('chiral', True)) sg_info = pointless.parse_pointless() except (autoprocess.errors.ProcessError, autoprocess.errors.ParserError, IOError) as e: return {'step': 'symmetry', 'success': False, 'reason': str(e)} xds_params = get_symmetry_params(sg_info['sg_number'], dset) sg_info.update(xds_params) _logger.info('Selected {}: {} , #{:d} '.format( sg_info['type'], xtal.SG_SYMBOLS[sg_info['sg_number']], sg_info['sg_number'], )) return {'step': 'symmetry', 'success': True, 'data': sg_info}
def harvest_correct(): if not misc.file_requirements('CORRECT.LP', 'GXPARM.XDS', 'XDS_ASCII.HKL'): return { 'step': 'integration', 'success': False, 'reason': 'Required files missing' } else: info = xds.parse_integrate() programs.xdsstat('XDS_ASCII.HKL') stat_info = xds.parse_xdsstat() info.update(stat_info) if info.get('failure') is None: info['output_file'] = 'XDS_ASCII.HKL' if len(info.get('statistics', [])) > 1 and info.get('summary') is not None: info['summary']['resolution'] = xtal.select_resolution( info['statistics']) return {'step': 'correction', 'success': True, 'data': info} else: return { 'step': 'correction', 'success': False, 'reason': info['failure'] }
def harvest_spots(): if misc.file_requirements('SPOT.XDS'): return {'step': 'spot_search', 'success': True} else: return { 'step': 'spot_search', 'success': False, 'reason': 'Could not find spots.' }
def data_quality(dset, options=None): filename = dset.results['scaling']['output_file'] options = options or {} os.chdir(options.get('directory', '.')) law_type = {'PM': 'Pseudo-merohedral', 'M': 'Merohedral'} # Check Requirements if not misc.file_requirements(filename): return { 'step': 'data_quality', 'success': False, 'reason': 'Required files missing' } try: programs.xtriage(filename, label=f'Checking quality of dataset "{dset.name}"') info = phenix.parse_xtriage() except autoprocess.errors.ProcessError as e: return {'step': 'data_quality', 'success': False, 'reason': str(e)} statistics_deviate = False if info.get('twinning_l_zscore', 0) > 3.5: statistics_deviate = True logger.warning( 'Intensity statistics deviate significantly from expected.') if len(info.get('twin_laws', [])) > 0: if statistics_deviate: logger.warning( 'Possible twin laws which may explain the deviation:') else: logger.info('Possible twin laws:') max_fraction = 0.0 for law in info['twin_laws']: fraction = 100.0 * max( [law['britton_alpha'], law['ML_alpha'], law['H_alpha']]) txt = ".. {} operator: [{}], Max twin-fraction: {:0.1f}%".format( law_type[law['type'].strip()], law['operator'].strip(), fraction, ) max_fraction = max(max_fraction, fraction) if statistics_deviate: logger.warning(txt) else: logger.info(txt) if not statistics_deviate and max_fraction > 10.0: logger.warning( 'Despite reasonable intensity statistics, high twin-fraction suggests wrong symmetry.' ) if statistics_deviate and len(info.get('twin_laws', [])) == 0: logger.warning( 'No pseudo/merohedral twin laws possible in this lattice.') return {'step': 'data_quality', 'success': True, 'data': info}
def harvest_initialize(): if misc.file_requirements('X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BKGINIT.cbf', 'BLANK.cbf', 'GAIN.cbf'): return {'step': 'initialize', 'success': True} else: return { 'step': 'initialize', 'success': False, 'reason': 'Initialization unsuccessful!' }
def calc_strategy(data_info, options=None): options = options or {} os.chdir(data_info['working_directory']) # indicate overwritten parameters suffix = [] if options.get('resolution'): suffix.append("res=%0.2f" % options.get('resolution')) if options.get('anomalous'): suffix.append("anomalous") if len(suffix) > 0: step_descr = "Calculating strategy [{}]".format(", ".join(suffix)) else: step_descr = 'Calculating strategy' if not misc.file_requirements('CORRECT.LP', 'BKGPIX.cbf', 'XDS_ASCII.HKL', 'GXPARM.XDS'): return { 'step': 'strategy', 'success': False, 'reason': 'Required files from integration missing' } if os.path.exists('GXPARM.XDS'): misc.backup_files('XPARM.XDS') shutil.copy('GXPARM.XDS', 'XPARM.XDS') run_info = { 'mode': options.get('mode'), 'anomalous': options.get('anomalous', False) } run_info.update(data_info) xdsio.write_xds_input("XPLAN", run_info) try: programs.xds_par(step_descr) info = xds.parse_xplan() programs.best(data_info, options) info.update(best.parse_best()) except autoprocess.errors.ProcessError as e: return { 'step': 'strategy', 'success': True, 'reason': str(e), 'data': info } return {'step': 'strategy', 'success': True, 'data': info}
def analyse_image(data_info, options=None): options = options or {} os.chdir(data_info['working_directory']) _logger.info('Analyzing reference image ...') try: programs.distl(data_info['reference_image']) except autoprocess.errors.ProcessError as e: return {'step': 'image_analysis', 'success': False, 'reason': str(e)} if not misc.file_requirements('distl.log'): return { 'step': 'image_analysis', 'success': False, 'reason': 'Could not analyse reference image' } info = distl.parse_distl('distl.log') return {'step': 'image_analysis', 'success': True, 'data': info}
def harvest_integrate(): if not misc.file_requirements('INTEGRATE.LP', 'CORRECT.LP'): return { 'step': 'integration', 'success': False, 'reason': 'Required files missing' } else: info = xds.parse_integrate() info['statistics'] = xds.parse_correct() if info.get('failure') is None: info['output_file'] = 'INTEGRATE.HKL' return {'step': 'integration', 'success': True, 'data': info} else: return { 'step': 'integration', 'success': False, 'reason': info['failure'] }
def solve_small_molecule(info, options=None): options = options or {} os.chdir(options.get('directory', '.')) _logger.info("Solving small-molecule structure ...") if not misc.file_requirements('%s-shelx.hkl' % info['name']): print(("File not found %s-shelx.hkl" % info['name'])) return { 'step': 'symmetry', 'success': False, 'reason': 'Required reflection files missing' } try: programs.shelx_sm(info['name'], info['unit_cell'], info['formula']) except (autoprocess.errors.ProcessError, autoprocess.errors.ParserError, IOError) as e: return {'step': 'symmetry', 'success': False, 'reason': str(e)} _smx_dir = os.path.relpath( os.path.join(options.get('directory', '.'), 'shelx-sm', info['name']), options.get('command_dir')) _logger.info('Coordinates: %s.res, Phases: %s.fcf' % (_smx_dir, _smx_dir)) return {'step': 'smx_structure', 'success': True, 'data': None}
def auto_index(data_info, options=None): options = options or {} os.chdir(data_info['working_directory']) step_descr = 'Determining lattice orientation and parameters' jobs = 'IDXREF' run_info = {'mode': options.get('mode')} run_info.update(data_info) if not misc.file_requirements('XDS.INP', 'SPOT.XDS'): return { 'step': 'indexing', 'success': False, 'reason': "Required files not found" } try: xdsio.write_xds_input(jobs, run_info) programs.xds_par(step_descr) info = xds.parse_idxref() diagnosis = diagnose_index(info) _retries = 0 sigma = 6 spot_size = 3 _aliens_removed = False _weak_removed = False _spot_adjusted = False while info.get('failure_code') > 0 and _retries < 8: _all_images = (run_info['spot_range'][0] == run_info['data_range']) _retries += 1 _logger.warning('Indexing failed:') for prob in diagnosis['problems']: _logger.warning('... {}'.format(PROBLEMS[prob])) if options.get('backup', False): misc.backup_files('SPOT.XDS', 'IDXREF.LP') if diagnosis['problems'] & {PROBLEMS.index_origin}: if not _all_images: step_descr = '-> Expanding Spot Range' run_info['spot_range'] = [run_info['data_range']] else: step_descr = '-> Adjusting detector origin' run_info['beam_center'] = diagnosis['options'].get( 'beam_center', run_info['beam_center']) xdsio.write_xds_input('COLSPOT IDXREF', run_info) programs.xds_par(step_descr) info = xds.parse_idxref() diagnosis = diagnose_index(info) elif (diagnosis['problems'] & {PROBLEMS.few_spots, PROBLEMS.dimension_2d }) and not _all_images: run_info.update(spot_range=[run_info['data_range']]) xdsio.write_xds_input('IDXREF', run_info) programs.xds_par('-> Expanding Spot Range') info = xds.parse_idxref() diagnosis = diagnose_index(info) elif (diagnosis['problems'] & { PROBLEMS.poor_solution, PROBLEMS.spot_accuracy, PROBLEMS.non_integral }) and not _spot_adjusted: spot_size *= 1.5 sigma = 6 new_params = { 'sigma': sigma, 'min_spot_size': spot_size, 'refine_index': "CELL BEAM ORIENTATION AXIS" } if not _all_images: new_params['spot_range'] = [run_info['data_range']] run_info.update(new_params) xdsio.write_xds_input('COLSPOT IDXREF', run_info) programs.xds_par( '-> Adjusting spot size and refinement parameters') info = xds.parse_idxref() diagnosis = diagnose_index(info) _spot_adjusted = spot_size > 12 elif (diagnosis['problems'] & {PROBLEMS.unindexed_spots}) and not _weak_removed: sigma += 3 _filter_spots(sigma=sigma) run_info.update(sigma=sigma) xdsio.write_xds_input('IDXREF', run_info) programs.xds_par( '-> Removing weak spots (Sigma < {:2.0f})'.format(sigma)) info = xds.parse_idxref() diagnosis = diagnose_index(info) _weak_removed = sigma >= 12 elif (diagnosis['problems'] & {PROBLEMS.unindexed_spots, PROBLEMS.multiple_subtrees }) and not _aliens_removed: _filter_spots(unindexed=True) xdsio.write_xds_input(jobs, run_info) programs.xds_par('-> Removing all alien spots') info = xds.parse_idxref() diagnosis = diagnose_index(info) _aliens_removed = True else: _logger.critical('.. Unable to proceed.') _retries = 999 except autoprocess.errors.ProcessError as e: return {'step': 'indexing', 'success': False, 'reason': str(e)} if info.get('failure_code') == 0: return {'step': 'indexing', 'success': True, 'data': info} else: return { 'step': 'indexing', 'success': False, 'reason': info['failure'] }
def convert_formats(dataset, options=None): options = options or {} os.chdir(options['directory']) # GENERATE MTZ and CNS output files infile = dataset.results['scaling'].get('output_file') out_file_dir = os.path.dirname(infile) out_file_base = os.path.basename(infile) out_file_root = os.path.join(out_file_dir, options.get('file_root', dataset.name)) output_files = [] logger.info( 'Generating MTZ, SHELX & CNS files from {} ...'.format(out_file_base)) if not misc.file_requirements( dataset.results['scaling'].get('output_file')): return { 'step': 'conversion', 'success': False, 'reason': 'Required files missing' } # Create convertion options conv_options = [{ 'resolution': 0, 'format': 'CNS', 'anomalous': options.get('anomalous', False), 'input_file': infile, 'output_file': out_file_root + ".cns", 'freeR_fraction': 0.05 }, { 'resolution': 0, 'format': 'SHELX', 'anomalous': options.get('anomalous', False), 'input_file': infile, 'output_file': out_file_root + "-shelx.hkl", 'freeR_fraction': 0, }, { 'resolution': 0, 'format': 'CCP4_I', 'anomalous': True, 'input_file': infile, 'output_file': out_file_root + ".ccp4i", 'freeR_fraction': 0.05, }] for opt in conv_options: try: if opt['format'] == 'CCP4_I': out_file = out_file_root + ".mtz" out_format = 'MTZ' else: out_file = opt['output_file'] out_format = opt['format'] xdsio.write_xdsconv_input(opt) label = f'Preparing {out_format} output file' programs.xdsconv(label=f'{label:<27}', final=log.TermColor.bold(out_file)) # Special formatting for MTZ if opt['format'] == 'CCP4_I': mtz_file = out_file_root + ".mtz" programs.f2mtz(mtz_file) output_files.append(mtz_file) else: output_files.append(opt['output_file']) except autoprocess.errors.ProcessError as e: logger.warning(f'Error creating {opt["format"]} file: {e}') if len(output_files) == 0: return { 'step': 'conversion', 'success': False, 'reason': 'No output files generated' } else: return {'step': 'conversion', 'success': True, 'data': output_files}
def integrate(data_info, options=None): options = {} if options is None else options os.chdir(data_info['working_directory']) run_info = {'mode': options.get('mode')} run_info.update(data_info) if options.get('backup', False): misc.backup_files('INTEGRATE.LP', 'INTEGRATE.HKL') # if optimizing the integration, copy GXPARM # Calculate actual number of frames full_range = list( range(run_info['data_range'][0], run_info['data_range'][1] + 1)) skip_ranges = [] for r_s, r_e in run_info['skip_range']: skip_ranges.extend(list(range(r_s, r_e + 1))) num_frames = len(set(full_range) - set(skip_ranges)) if options.get('optimize', False) and os.path.exists('GXPARM.XDS'): misc.backup_files('XPARM.XDS') shutil.copy('GXPARM.XDS', 'XPARM.XDS') step_descr = 'Optimizing {:d} frames of dataset {}'.format( num_frames, log.TermColor.italics(data_info['name'])) else: step_descr = 'Integrating {:d} frames of dataset {}'.format( num_frames, log.TermColor.italics(data_info['name'])) # check if we are screening screening = options.get('mode') == 'screen' xdsio.write_xds_input("DEFPIX INTEGRATE", run_info) if not misc.file_requirements('X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'XPARM.XDS'): return { 'step': 'integration', 'success': False, 'reason': 'Required files missing' } try: programs.xds_par(step_descr) info = xds.parse_integrate() except autoprocess.errors.ProcessError as e: return {'step': 'integration', 'success': False, 'reason': str(e)} except: return { 'step': 'integration', 'success': False, 'reason': "Could not parse integrate output file" } else: pass if info.get('failure') is None: if data_info['working_directory'] == options.get('directory'): info['output_file'] = 'INTEGRATE.HKL' else: info['output_file'] = os.path.join(data_info['name'], 'INTEGRATE.HKL') return {'step': 'integration', 'success': True, 'data': info} else: return { 'step': 'integration', 'success': False, 'reason': info['failure'] }
def correct(data_info, options=None): options = options or {} os.chdir(data_info['working_directory']) message = options.get('message', "Applying corrections to") step_descr = '{} dataset "{}" for space-group {}'.format( message, data_info['name'], xtal.SG_SYMBOLS[data_info['space_group']]) run_info = {'mode': options.get('mode')} run_info.update(data_info) if not misc.file_requirements('INTEGRATE.HKL', 'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf'): return { 'step': 'correction', 'success': False, 'reason': 'Required files missing' } if options.get('backup', False): misc.backup_files('XDS_ASCII.HKL', 'CORRECT.LP') xdsio.write_xds_input("CORRECT", run_info) try: programs.xds_par(step_descr) info = xds.parse_correct() # enable correction factors if anomalous data and repeat correction if info.get('correction_factors') is not None and options.get( 'anomalous', False): for f in info['correction_factors'].get('factors', []): if abs(f['chi_sq_fit'] - 1.0) > 0.25: run_info.update({'strict_absorption': True}) xdsio.write_xds_input("CORRECT", run_info) programs.xds_par() info = xds.parse_correct() info['strict_absorption'] = True break # Extra statistics if data_info['working_directory'] == options.get('directory'): info['output_file'] = 'XDS_ASCII.HKL' else: sub_dir = os.path.relpath(data_info['working_directory'], options.get('directory', '')) info['output_file'] = os.path.join(sub_dir, 'XDS_ASCII.HKL') programs.xdsstat('XDS_ASCII.HKL') stat_info = xds.parse_xdsstat() info.update(stat_info) except autoprocess.errors.ProcessError as e: return {'step': 'correction', 'success': False, 'reason': str(e)} if info.get('failure') is None: if len(info.get('statistics', [])) > 1 and info.get('summary') is not None: info['summary']['resolution'] = xtal.select_resolution( info['statistics']) return {'step': 'correction', 'success': True, 'data': info} else: return { 'step': 'correction', 'success': False, 'reason': info['failure'] }