def run(args=None, phil=working_phil):
"""
Set up refinement from command line options, files and PHIL parameters.
Run refinement and save output files as specified.
Called when running dials.refine as a command-line program
Args:
args (list): Additional command-line arguments
phil: The working PHIL parameters
Returns:
None
"""
# The script usage
usage = (
"usage: dials.refine [options] [param.phil] " "models.expt observations.refl"
)
# Create the parser
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
# Parse the command line
params, options = parser.parse_args(args=args, show_diff_phil=False)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments
)
# Configure the logging
dials.util.log.config(verbosity=options.verbose, logfile=params.output.log)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0 or len(reflections) == 0:
parser.print_help()
return
if len(reflections) > 1:
sys.exit("Only one reflections list can be imported at present")
reflections = reflections[0]
# check input is suitable
msg = (
"The supplied reflection table does not have the required data " + "column: {0}"
)
for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
if key not in reflections:
sys.exit(msg.format(key))
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Warn about potentially unhelpful options
if params.refinement.mp.nproc > 1:
logger.warning(
"Setting nproc > 1 is only helpful in rare "
"circumstances. It is not recommended for typical data processing "
"tasks."
)
if params.refinement.parameterisation.scan_varying is not False:
# duplicate crystal if necessary for scan varying - will need
# to compare the scans with crystals - if not 1:1 will need to
# split the crystals
crystal_has_scan = {}
for j, e in enumerate(experiments):
if e.crystal in crystal_has_scan:
if e.scan is not crystal_has_scan[e.crystal]:
logger.info(
"Duplicating crystal model for scan-varying refinement of experiment %d"
% j
)
e.crystal = copy.deepcopy(e.crystal)
else:
crystal_has_scan[e.crystal] = e.scan
# Run refinement
experiments, reflections, refiner, history = run_dials_refine(
experiments, reflections, params
)
# For the usual case of refinement of one crystal, print that model for information
crystals = experiments.crystals()
if len(crystals) == 1:
logger.info("")
logger.info("Final refined crystal model:")
logger.info(crystals[0])
# Write table of centroids to file, if requested
if params.output.centroids:
logger.info(
"Writing table of centroids to '{}'".format(params.output.centroids)
)
write_centroids_table(refiner, params.output.centroids)
# Write scan-varying parameters to file, if there were any
if params.output.parameter_table:
scans = experiments.scans()
if len(scans) > 1:
logger.info(
"Writing a scan-varying parameter table is only supported "
"for refinement of a single scan"
)
else:
scan = scans[0]
text = refiner.get_param_reporter().varying_params_vs_image_number(
scan.get_array_range()
)
if text:
logger.info(
"Writing scan-varying parameter table to {}".format(
params.output.parameter_table
)
)
f = open(params.output.parameter_table, "w")
f.write(text)
f.close()
else:
logger.info("No scan-varying parameter table to write")
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info("Saving refined experiments to {}".format(output_experiments_filename))
experiments.as_file(output_experiments_filename)
# Save reflections with updated predictions if requested (allow to switch
# this off if it is a time-consuming step)
if params.output.reflections:
logger.info(
"Saving reflections with updated predictions to {}".format(
params.output.reflections
)
)
if params.output.include_unused_reflections:
reflections.as_file(params.output.reflections)
else:
sel = reflections.get_flags(reflections.flags.used_in_refinement)
reflections.select(sel).as_file(params.output.reflections)
# Save matches to file for debugging
if params.output.matches:
matches = refiner.get_matches()
logger.info(
"Saving matches (use for debugging purposes) to {}".format(
params.output.matches
)
)
matches.as_file(params.output.matches)
# Create correlation plots
if params.output.correlation_plot.filename is not None:
create_correlation_plots(refiner, params.output)
# Save refinement history
if params.output.history:
logger.info(
"Saving refinement step history to {}".format(params.output.history)
)
history.to_json_file(params.output.history)
def run(self):
"""Execute the script."""
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
from dxtbx.model.experiment_list import ExperimentList
experiments = ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(
params.input.reflections, params.input.experiments
):
refs = ref_wrapper.data
exps = exp_wrapper.data
for i, exp in enumerate(exps):
sel = refs["id"] == i
sub_ref = refs.select(sel)
nrefs_per_exp.append(len(sub_ref))
sub_ref["id"] = flex.int(len(sub_ref), global_id)
reflections.extend(sub_ref)
experiments.append(exp)
global_id += 1
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print("No Experiments found in the input")
self.parser.print_help()
return
if len(reflections) == 0:
print("No reflection data found in the input")
self.parser.print_help()
return
self.check_input(reflections)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Convert to P 1?
if params.refinement.triclinic:
reflections, experiments = self.convert_to_P1(reflections, experiments)
# Combine crystals?
if params.refinement.combine_crystal_models and len(experiments) > 1:
logger.info("Combining {0} crystal models".format(len(experiments)))
experiments = self.combine_crystals(experiments)
# Filter integrated centroids?
if params.refinement.filter_integrated_centroids:
reflections = self.filter_integrated_centroids(reflections)
# Get the refiner
logger.info("Configuring refiner")
refiner = self.create_refiner(params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info("Performing refinement of a single Experiment...")
else:
logger.info("Performing refinement of {} Experiments...".format(nexp))
refiner.run()
# get the refined experiments
experiments = refiner.get_experiments()
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info("")
logger.info("Final refined crystal model:")
logger.info(crystals[0])
logger.info(self.cell_param_table(crystals[0]))
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info(
"Saving refined experiments to {}".format(output_experiments_filename)
)
from dxtbx.model.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Create correlation plots
if params.output.correlation_plot.filename is not None:
create_correlation_plots(refiner, params.output)
if params.output.cif is not None:
self.generate_cif(crystals[0], refiner, file=params.output.cif)
if params.output.p4p is not None:
self.generate_p4p(crystals[0], experiments[0].beam, file=params.output.p4p)
if params.output.mmcif is not None:
self.generate_mmcif(crystals[0], refiner, file=params.output.mmcif)
# Log the total time taken
logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))
def run(self):
"""Execute the script."""
start_time = time()
# Parse the command line
params, _ = self.parser.parse_args(show_diff_phil=False)
# set up global reflections list
reflections = flex.reflection_table()
# loop through the input, building up the global lists
reflections_list = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
reflections_list = parse_multiple_datasets(reflections_list)
for refs in reflections_list:
reflections.extend(refs)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print("No Experiments found in the input")
self.parser.print_help()
return
if not reflections:
print("No reflection data found in the input")
self.parser.print_help()
return
self.check_input(reflections)
# Configure the logging
log.config(logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Convert to P 1?
if params.refinement.triclinic:
reflections, experiments = self.convert_to_P1(reflections, experiments)
# Combine crystals?
if params.refinement.combine_crystal_models and len(experiments) > 1:
logger.info("Combining {0} crystal models".format(len(experiments)))
experiments = self.combine_crystals(experiments)
# Filter integrated centroids?
if params.refinement.filter_integrated_centroids:
reflections = self.filter_integrated_centroids(reflections)
# Filter data if scaled to remove outliers
if "inverse_scale_factor" in reflections:
try:
reflections = filter_reflection_table(reflections, ["scale"])
except ValueError as e:
logger.warn(e)
logger.info(
"Filtering on scaled data failed, proceeding with integrated data."
)
# Get the refiner
logger.info("Configuring refiner")
refiner = self.create_refiner(params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info("Performing refinement of a single Experiment...")
else:
logger.info("Performing refinement of {} Experiments...".format(nexp))
refiner.run()
# get the refined experiments
experiments = refiner.get_experiments()
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info("")
logger.info("Final refined crystal model:")
logger.info(crystals[0])
logger.info(self.cell_param_table(crystals[0]))
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info(
"Saving refined experiments to {}".format(output_experiments_filename)
)
experiments.as_file(output_experiments_filename)
# Create correlation plots
if params.output.correlation_plot.filename is not None:
create_correlation_plots(refiner, params.output)
if params.output.cif is not None:
self.generate_cif(crystals[0], refiner, filename=params.output.cif)
if params.output.p4p is not None:
self.generate_p4p(
crystals[0], experiments[0].beam, filename=params.output.p4p
)
if params.output.mmcif is not None:
self.generate_mmcif(crystals[0], refiner, filename=params.output.mmcif)
# Log the total time taken
logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))
def run(args=None, phil=working_phil):
"""
Set up refinement from command line options, files and PHIL parameters.
Run refinement and save output files as specified.
Called when running dials.refine as a command-line program
Args:
args (list): Additional command-line arguments
phil: The working PHIL parameters
Returns:
None
"""
import dials.util.log
from dials.util.options import OptionParser
from dials.util.options import flatten_reflections
from dials.util.options import flatten_experiments
import libtbx.load_env
start_time = time()
# The script usage
usage = ("usage: %s [options] [param.phil] "
"models.expt observations.refl" % libtbx.env.dispatcher_name)
# Create the parser
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
# Parse the command line
params, options = parser.parse_args(args=args, show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
# Configure the logging
dials.util.log.config(verbosity=options.verbose, logfile=params.output.log)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
sys.exit("No Experiments found in the input")
if len(reflections) == 0:
sys.exit("No reflection data found in the input")
if len(reflections) > 1:
sys.exit("Only one reflections list can be imported at present")
reflections = reflections[0]
# check input is suitable
msg = ("The supplied reflection table does not have the required data " +
"column: {0}")
for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
if key not in reflections:
msg = msg.format(key)
raise dials.util.Sorry(msg)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Warn about potentially unhelpful options
if params.refinement.mp.nproc > 1:
logger.warning(
"WARNING: setting nproc > 1 is only helpful in rare "
"circumstances. It is not recommended for typical data processing "
"tasks.\n")
# Run refinement
experiments, reflections, refiner, history = run_dials_refine(
experiments, reflections, params)
# For the usual case of refinement of one crystal, print that model for information
crystals = experiments.crystals()
if len(crystals) == 1:
logger.info("")
logger.info("Final refined crystal model:")
logger.info(crystals[0])
# Write table of centroids to file, if requested
if params.output.centroids:
logger.info("Writing table of centroids to '{}'".format(
params.output.centroids))
write_centroids_table(refiner, params.output.centroids)
# Write scan-varying parameters to file, if there were any
if params.output.parameter_table:
scans = experiments.scans()
if len(scans) > 1:
logger.info(
"Writing a scan-varying parameter table is only supported "
"for refinement of a single scan")
else:
scan = scans[0]
text = refiner.get_param_reporter().varying_params_vs_image_number(
scan.get_array_range())
if text:
logger.info(
"Writing scan-varying parameter table to {}".format(
params.output.parameter_table))
f = open(params.output.parameter_table, "w")
f.write(text)
f.close()
else:
logger.info("No scan-varying parameter table to write")
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info(
"Saving refined experiments to {}".format(output_experiments_filename))
from dxtbx.model.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments_filename)
# Save reflections with updated predictions if requested (allow to switch
# this off if it is a time-consuming step)
if params.output.reflections:
logger.info("Saving reflections with updated predictions to {}".format(
params.output.reflections))
if params.output.include_unused_reflections:
reflections.as_file(params.output.reflections)
else:
sel = reflections.get_flags(reflections.flags.used_in_refinement)
reflections.select(sel).as_file(params.output.reflections)
# Save matches to file for debugging
if params.output.matches:
matches = refiner.get_matches()
logger.info("Saving matches (use for debugging purposes) to {}".format(
params.output.matches))
matches.as_file(params.output.matches)
# Create correlation plots
if params.output.correlation_plot.filename is not None:
create_correlation_plots(refiner, params.output)
# Save refinement history
if params.output.history:
logger.info("Saving refinement step history to {}".format(
params.output.history))
history.to_json_file(params.output.history)
# Log the total time taken
logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))
