def determine_sg(data_info, dset, options=None):
options = options or {}
os.chdir(data_info['working_directory'])
_logger.info("Automaticaly Determining Symmetry ...")
if not misc.file_requirements('INTEGRATE.HKL'):
return {
'step': 'symmetry',
'success': False,
'reason': 'Required files from integration missing'
}
try:
programs.pointless(chiral=options.get('chiral', True))
sg_info = pointless.parse_pointless()
except (autoprocess.errors.ProcessError, autoprocess.errors.ParserError,
IOError) as e:
return {'step': 'symmetry', 'success': False, 'reason': str(e)}
xds_params = get_symmetry_params(sg_info['sg_number'], dset)
sg_info.update(xds_params)
_logger.info('Selected {}: {} , #{:d} '.format(
sg_info['type'],
xtal.SG_SYMBOLS[sg_info['sg_number']],
sg_info['sg_number'],
))
return {'step': 'symmetry', 'success': True, 'data': sg_info}
def harvest_correct():
if not misc.file_requirements('CORRECT.LP', 'GXPARM.XDS', 'XDS_ASCII.HKL'):
return {
'step': 'integration',
'success': False,
'reason': 'Required files missing'
}
else:
info = xds.parse_integrate()
programs.xdsstat('XDS_ASCII.HKL')
stat_info = xds.parse_xdsstat()
info.update(stat_info)
if info.get('failure') is None:
info['output_file'] = 'XDS_ASCII.HKL'
if len(info.get('statistics',
[])) > 1 and info.get('summary') is not None:
info['summary']['resolution'] = xtal.select_resolution(
info['statistics'])
return {'step': 'correction', 'success': True, 'data': info}
else:
return {
'step': 'correction',
'success': False,
'reason': info['failure']
}
def harvest_spots():
if misc.file_requirements('SPOT.XDS'):
return {'step': 'spot_search', 'success': True}
else:
return {
'step': 'spot_search',
'success': False,
'reason': 'Could not find spots.'
}
def data_quality(dset, options=None):
filename = dset.results['scaling']['output_file']
options = options or {}
os.chdir(options.get('directory', '.'))
law_type = {'PM': 'Pseudo-merohedral', 'M': 'Merohedral'}
# Check Requirements
if not misc.file_requirements(filename):
return {
'step': 'data_quality',
'success': False,
'reason': 'Required files missing'
}
try:
programs.xtriage(filename,
label=f'Checking quality of dataset "{dset.name}"')
info = phenix.parse_xtriage()
except autoprocess.errors.ProcessError as e:
return {'step': 'data_quality', 'success': False, 'reason': str(e)}
statistics_deviate = False
if info.get('twinning_l_zscore', 0) > 3.5:
statistics_deviate = True
logger.warning(
'Intensity statistics deviate significantly from expected.')
if len(info.get('twin_laws', [])) > 0:
if statistics_deviate:
logger.warning(
'Possible twin laws which may explain the deviation:')
else:
logger.info('Possible twin laws:')
max_fraction = 0.0
for law in info['twin_laws']:
fraction = 100.0 * max(
[law['britton_alpha'], law['ML_alpha'], law['H_alpha']])
txt = ".. {} operator: [{}], Max twin-fraction: {:0.1f}%".format(
law_type[law['type'].strip()],
law['operator'].strip(),
fraction,
)
max_fraction = max(max_fraction, fraction)
if statistics_deviate:
logger.warning(txt)
else:
logger.info(txt)
if not statistics_deviate and max_fraction > 10.0:
logger.warning(
'Despite reasonable intensity statistics, high twin-fraction suggests wrong symmetry.'
)
if statistics_deviate and len(info.get('twin_laws', [])) == 0:
logger.warning(
'No pseudo/merohedral twin laws possible in this lattice.')
return {'step': 'data_quality', 'success': True, 'data': info}
def harvest_initialize():
if misc.file_requirements('X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf',
'BKGINIT.cbf', 'BLANK.cbf', 'GAIN.cbf'):
return {'step': 'initialize', 'success': True}
else:
return {
'step': 'initialize',
'success': False,
'reason': 'Initialization unsuccessful!'
}
def calc_strategy(data_info, options=None):
options = options or {}
os.chdir(data_info['working_directory'])
# indicate overwritten parameters
suffix = []
if options.get('resolution'):
suffix.append("res=%0.2f" % options.get('resolution'))
if options.get('anomalous'):
suffix.append("anomalous")
if len(suffix) > 0:
step_descr = "Calculating strategy [{}]".format(", ".join(suffix))
else:
step_descr = 'Calculating strategy'
if not misc.file_requirements('CORRECT.LP', 'BKGPIX.cbf', 'XDS_ASCII.HKL',
'GXPARM.XDS'):
return {
'step': 'strategy',
'success': False,
'reason': 'Required files from integration missing'
}
if os.path.exists('GXPARM.XDS'):
misc.backup_files('XPARM.XDS')
shutil.copy('GXPARM.XDS', 'XPARM.XDS')
run_info = {
'mode': options.get('mode'),
'anomalous': options.get('anomalous', False)
}
run_info.update(data_info)
xdsio.write_xds_input("XPLAN", run_info)
try:
programs.xds_par(step_descr)
info = xds.parse_xplan()
programs.best(data_info, options)
info.update(best.parse_best())
except autoprocess.errors.ProcessError as e:
return {
'step': 'strategy',
'success': True,
'reason': str(e),
'data': info
}
return {'step': 'strategy', 'success': True, 'data': info}
def analyse_image(data_info, options=None):
options = options or {}
os.chdir(data_info['working_directory'])
_logger.info('Analyzing reference image ...')
try:
programs.distl(data_info['reference_image'])
except autoprocess.errors.ProcessError as e:
return {'step': 'image_analysis', 'success': False, 'reason': str(e)}
if not misc.file_requirements('distl.log'):
return {
'step': 'image_analysis',
'success': False,
'reason': 'Could not analyse reference image'
}
info = distl.parse_distl('distl.log')
return {'step': 'image_analysis', 'success': True, 'data': info}
def harvest_integrate():
if not misc.file_requirements('INTEGRATE.LP', 'CORRECT.LP'):
return {
'step': 'integration',
'success': False,
'reason': 'Required files missing'
}
else:
info = xds.parse_integrate()
info['statistics'] = xds.parse_correct()
if info.get('failure') is None:
info['output_file'] = 'INTEGRATE.HKL'
return {'step': 'integration', 'success': True, 'data': info}
else:
return {
'step': 'integration',
'success': False,
'reason': info['failure']
}
def solve_small_molecule(info, options=None):
options = options or {}
os.chdir(options.get('directory', '.'))
_logger.info("Solving small-molecule structure ...")
if not misc.file_requirements('%s-shelx.hkl' % info['name']):
print(("File not found %s-shelx.hkl" % info['name']))
return {
'step': 'symmetry',
'success': False,
'reason': 'Required reflection files missing'
}
try:
programs.shelx_sm(info['name'], info['unit_cell'], info['formula'])
except (autoprocess.errors.ProcessError, autoprocess.errors.ParserError,
IOError) as e:
return {'step': 'symmetry', 'success': False, 'reason': str(e)}
_smx_dir = os.path.relpath(
os.path.join(options.get('directory', '.'), 'shelx-sm', info['name']),
options.get('command_dir'))
_logger.info('Coordinates: %s.res, Phases: %s.fcf' % (_smx_dir, _smx_dir))
return {'step': 'smx_structure', 'success': True, 'data': None}
def auto_index(data_info, options=None):
options = options or {}
os.chdir(data_info['working_directory'])
step_descr = 'Determining lattice orientation and parameters'
jobs = 'IDXREF'
run_info = {'mode': options.get('mode')}
run_info.update(data_info)
if not misc.file_requirements('XDS.INP', 'SPOT.XDS'):
return {
'step': 'indexing',
'success': False,
'reason': "Required files not found"
}
try:
xdsio.write_xds_input(jobs, run_info)
programs.xds_par(step_descr)
info = xds.parse_idxref()
diagnosis = diagnose_index(info)
_retries = 0
sigma = 6
spot_size = 3
_aliens_removed = False
_weak_removed = False
_spot_adjusted = False
while info.get('failure_code') > 0 and _retries < 8:
_all_images = (run_info['spot_range'][0] == run_info['data_range'])
_retries += 1
_logger.warning('Indexing failed:')
for prob in diagnosis['problems']:
_logger.warning('... {}'.format(PROBLEMS[prob]))
if options.get('backup', False):
misc.backup_files('SPOT.XDS', 'IDXREF.LP')
if diagnosis['problems'] & {PROBLEMS.index_origin}:
if not _all_images:
step_descr = '-> Expanding Spot Range'
run_info['spot_range'] = [run_info['data_range']]
else:
step_descr = '-> Adjusting detector origin'
run_info['beam_center'] = diagnosis['options'].get(
'beam_center', run_info['beam_center'])
xdsio.write_xds_input('COLSPOT IDXREF', run_info)
programs.xds_par(step_descr)
info = xds.parse_idxref()
diagnosis = diagnose_index(info)
elif (diagnosis['problems']
& {PROBLEMS.few_spots, PROBLEMS.dimension_2d
}) and not _all_images:
run_info.update(spot_range=[run_info['data_range']])
xdsio.write_xds_input('IDXREF', run_info)
programs.xds_par('-> Expanding Spot Range')
info = xds.parse_idxref()
diagnosis = diagnose_index(info)
elif (diagnosis['problems'] & {
PROBLEMS.poor_solution, PROBLEMS.spot_accuracy,
PROBLEMS.non_integral
}) and not _spot_adjusted:
spot_size *= 1.5
sigma = 6
new_params = {
'sigma': sigma,
'min_spot_size': spot_size,
'refine_index': "CELL BEAM ORIENTATION AXIS"
}
if not _all_images:
new_params['spot_range'] = [run_info['data_range']]
run_info.update(new_params)
xdsio.write_xds_input('COLSPOT IDXREF', run_info)
programs.xds_par(
'-> Adjusting spot size and refinement parameters')
info = xds.parse_idxref()
diagnosis = diagnose_index(info)
_spot_adjusted = spot_size > 12
elif (diagnosis['problems']
& {PROBLEMS.unindexed_spots}) and not _weak_removed:
sigma += 3
_filter_spots(sigma=sigma)
run_info.update(sigma=sigma)
xdsio.write_xds_input('IDXREF', run_info)
programs.xds_par(
'-> Removing weak spots (Sigma < {:2.0f})'.format(sigma))
info = xds.parse_idxref()
diagnosis = diagnose_index(info)
_weak_removed = sigma >= 12
elif (diagnosis['problems']
& {PROBLEMS.unindexed_spots, PROBLEMS.multiple_subtrees
}) and not _aliens_removed:
_filter_spots(unindexed=True)
xdsio.write_xds_input(jobs, run_info)
programs.xds_par('-> Removing all alien spots')
info = xds.parse_idxref()
diagnosis = diagnose_index(info)
_aliens_removed = True
else:
_logger.critical('.. Unable to proceed.')
_retries = 999
except autoprocess.errors.ProcessError as e:
return {'step': 'indexing', 'success': False, 'reason': str(e)}
if info.get('failure_code') == 0:
return {'step': 'indexing', 'success': True, 'data': info}
else:
return {
'step': 'indexing',
'success': False,
'reason': info['failure']
}
def convert_formats(dataset, options=None):
options = options or {}
os.chdir(options['directory'])
# GENERATE MTZ and CNS output files
infile = dataset.results['scaling'].get('output_file')
out_file_dir = os.path.dirname(infile)
out_file_base = os.path.basename(infile)
out_file_root = os.path.join(out_file_dir,
options.get('file_root', dataset.name))
output_files = []
logger.info(
'Generating MTZ, SHELX & CNS files from {} ...'.format(out_file_base))
if not misc.file_requirements(
dataset.results['scaling'].get('output_file')):
return {
'step': 'conversion',
'success': False,
'reason': 'Required files missing'
}
# Create convertion options
conv_options = [{
'resolution': 0,
'format': 'CNS',
'anomalous': options.get('anomalous', False),
'input_file': infile,
'output_file': out_file_root + ".cns",
'freeR_fraction': 0.05
}, {
'resolution': 0,
'format': 'SHELX',
'anomalous': options.get('anomalous', False),
'input_file': infile,
'output_file': out_file_root + "-shelx.hkl",
'freeR_fraction': 0,
}, {
'resolution': 0,
'format': 'CCP4_I',
'anomalous': True,
'input_file': infile,
'output_file': out_file_root + ".ccp4i",
'freeR_fraction': 0.05,
}]
for opt in conv_options:
try:
if opt['format'] == 'CCP4_I':
out_file = out_file_root + ".mtz"
out_format = 'MTZ'
else:
out_file = opt['output_file']
out_format = opt['format']
xdsio.write_xdsconv_input(opt)
label = f'Preparing {out_format} output file'
programs.xdsconv(label=f'{label:<27}',
final=log.TermColor.bold(out_file))
# Special formatting for MTZ
if opt['format'] == 'CCP4_I':
mtz_file = out_file_root + ".mtz"
programs.f2mtz(mtz_file)
output_files.append(mtz_file)
else:
output_files.append(opt['output_file'])
except autoprocess.errors.ProcessError as e:
logger.warning(f'Error creating {opt["format"]} file: {e}')
if len(output_files) == 0:
return {
'step': 'conversion',
'success': False,
'reason': 'No output files generated'
}
else:
return {'step': 'conversion', 'success': True, 'data': output_files}
def integrate(data_info, options=None):
options = {} if options is None else options
os.chdir(data_info['working_directory'])
run_info = {'mode': options.get('mode')}
run_info.update(data_info)
if options.get('backup', False):
misc.backup_files('INTEGRATE.LP', 'INTEGRATE.HKL')
# if optimizing the integration, copy GXPARM
# Calculate actual number of frames
full_range = list(
range(run_info['data_range'][0], run_info['data_range'][1] + 1))
skip_ranges = []
for r_s, r_e in run_info['skip_range']:
skip_ranges.extend(list(range(r_s, r_e + 1)))
num_frames = len(set(full_range) - set(skip_ranges))
if options.get('optimize', False) and os.path.exists('GXPARM.XDS'):
misc.backup_files('XPARM.XDS')
shutil.copy('GXPARM.XDS', 'XPARM.XDS')
step_descr = 'Optimizing {:d} frames of dataset {}'.format(
num_frames, log.TermColor.italics(data_info['name']))
else:
step_descr = 'Integrating {:d} frames of dataset {}'.format(
num_frames, log.TermColor.italics(data_info['name']))
# check if we are screening
screening = options.get('mode') == 'screen'
xdsio.write_xds_input("DEFPIX INTEGRATE", run_info)
if not misc.file_requirements('X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf',
'XPARM.XDS'):
return {
'step': 'integration',
'success': False,
'reason': 'Required files missing'
}
try:
programs.xds_par(step_descr)
info = xds.parse_integrate()
except autoprocess.errors.ProcessError as e:
return {'step': 'integration', 'success': False, 'reason': str(e)}
except:
return {
'step': 'integration',
'success': False,
'reason': "Could not parse integrate output file"
}
else:
pass
if info.get('failure') is None:
if data_info['working_directory'] == options.get('directory'):
info['output_file'] = 'INTEGRATE.HKL'
else:
info['output_file'] = os.path.join(data_info['name'],
'INTEGRATE.HKL')
return {'step': 'integration', 'success': True, 'data': info}
else:
return {
'step': 'integration',
'success': False,
'reason': info['failure']
}
def correct(data_info, options=None):
options = options or {}
os.chdir(data_info['working_directory'])
message = options.get('message', "Applying corrections to")
step_descr = '{} dataset "{}" for space-group {}'.format(
message, data_info['name'], xtal.SG_SYMBOLS[data_info['space_group']])
run_info = {'mode': options.get('mode')}
run_info.update(data_info)
if not misc.file_requirements('INTEGRATE.HKL', 'X-CORRECTIONS.cbf',
'Y-CORRECTIONS.cbf'):
return {
'step': 'correction',
'success': False,
'reason': 'Required files missing'
}
if options.get('backup', False):
misc.backup_files('XDS_ASCII.HKL', 'CORRECT.LP')
xdsio.write_xds_input("CORRECT", run_info)
try:
programs.xds_par(step_descr)
info = xds.parse_correct()
# enable correction factors if anomalous data and repeat correction
if info.get('correction_factors') is not None and options.get(
'anomalous', False):
for f in info['correction_factors'].get('factors', []):
if abs(f['chi_sq_fit'] - 1.0) > 0.25:
run_info.update({'strict_absorption': True})
xdsio.write_xds_input("CORRECT", run_info)
programs.xds_par()
info = xds.parse_correct()
info['strict_absorption'] = True
break
# Extra statistics
if data_info['working_directory'] == options.get('directory'):
info['output_file'] = 'XDS_ASCII.HKL'
else:
sub_dir = os.path.relpath(data_info['working_directory'],
options.get('directory', ''))
info['output_file'] = os.path.join(sub_dir, 'XDS_ASCII.HKL')
programs.xdsstat('XDS_ASCII.HKL')
stat_info = xds.parse_xdsstat()
info.update(stat_info)
except autoprocess.errors.ProcessError as e:
return {'step': 'correction', 'success': False, 'reason': str(e)}
if info.get('failure') is None:
if len(info.get('statistics',
[])) > 1 and info.get('summary') is not None:
info['summary']['resolution'] = xtal.select_resolution(
info['statistics'])
return {'step': 'correction', 'success': True, 'data': info}
else:
return {
'step': 'correction',
'success': False,
'reason': info['failure']
}