class Script(object):
''' A class to encapsulate the script. '''
def __init__(self):
''' Initialise the script. '''
from dials.util.options import OptionParser
import libtbx.load_env
# Create the parser
usage = "usage: %s [options] reflections.pickle" % libtbx.env.dispatcher_name
self.parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
epilog=help_message)
self.parser.add_option(
'--xkcd',
action='store_true',
dest='xkcd',
default=False,
help='Special drawing mode')
def run(self):
''' Run the script. '''
from dials.util.command_line import Command
from libtbx.utils import Sorry
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
if options.xkcd:
from matplotlib import pyplot
pyplot.xkcd()
# Shoe the help
if len(params.input.reflections) != 1:
self.parser.print_help()
exit(0)
# Analyse the reflections
analyse = Analyser(
params.output.directory, grid_size=params.grid_size,
pixels_per_bin=params.pixels_per_bin,
centroid_diff_max=params.centroid_diff_max)
analyse(params.input.reflections[0].data)
def run():
from dials.util.options import OptionParser
# Create the parser
usage = "usage: dials.analyse_output [options] observations.refl"
parser = OptionParser(usage=usage,
phil=phil_scope,
read_reflections=True,
epilog=help_message)
parser.add_option(
"--xkcd",
action="store_true",
dest="xkcd",
default=False,
help="Special drawing mode",
)
# Parse the command line arguments
params, options = parser.parse_args(show_diff_phil=True)
if options.xkcd:
pyplot.xkcd()
# Show the help
if len(params.input.reflections) != 1:
parser.print_help()
print("""
dials.analyse_output is deprecated and will be removed in a future release.
Its successor is dials.report, which generates interactive reports.
""")
exit(0)
# Analyse the reflections
analyse(
params.input.reflections[0].data,
params.output.directory,
grid_size=params.grid_size,
pixels_per_bin=params.pixels_per_bin,
centroid_diff_max=params.centroid_diff_max,
)
print("""
dials.analyse_output is deprecated and will be removed in a future release.
Its successor is dials.report, which generates interactive reports.
""")
class Script(object):
'''A class for running the script.'''
def __init__(self):
# The script usage
import libtbx.load_env
self.usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name
self.parser = None
def initialize(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
# Create the parser
self.parser = OptionParser(
usage=self.usage,
phil=phil_scope,
epilog=help_message)
self.parser.add_option(
'--plots',
action='store_true',
default=False,
dest='show_plots',
help='Show some plots.')
# Parse the command line. quick_parse is required for MPI compatibility
params, options = self.parser.parse_args(show_diff_phil=True,quick_parse=True)
self.params = params
self.options = options
def validate(self):
from xfel.merging.application.validation.application import application
application(self.params)
def modify(self, experiments, reflections):
return experiments, reflections #nop
def run(self):
print('''Initializing and validating phil...''')
self.initialize()
self.validate()
# do other stuff
return
class Script(base_Script):
def validate(self, comm):
base_Script.validate(self)
if (self.params.rescale_with_average_cell):
raise Usage("""Rescaling_with_average_cell not supported with MPI
(Would require a second round of scaling, inefficient).""")
if (self.params.mpi.cs == True and comm.Get_size() <= 1):
raise Usage(
"Client-server algorithm requires a minimum size of 2 MPI ranks to process data. Currently size=%d"
% comm.Get_size())
def initialize(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
from iotbx.phil import parse
from xfel.command_line.cxi_merge import master_phil
help_message = '''
MPI enabled script for merging xfel data
'''
phil_mpi = """
mpi {
logging = False
.type = bool
.help = logging Used to report timing information on the MPI operations. All strings are of the format '~<UNIQUE_STRING> START/END RANK=<MPI_RANK> TIME=<Unix time>;'
cs = False
.type = bool
.help = cs This enables the client-server work distribution duing merging. A minimum of 2 ranks are needed for this processing method.
.help = Default operation is set to false, which chunks the data evenly over all available ranks.
}
"""
phil_scope = parse(master_phil + phil_mpi)
# Create the parser
self.parser = OptionParser(usage=self.usage,
phil=phil_scope,
epilog=help_message)
self.parser.add_option('--plots',
action='store_true',
default=False,
dest='show_plots',
help='Show some plots.')
# Parse the command line. quick_parse is required for MPI compatibility
params, options = self.parser.parse_args(show_diff_phil=True,
quick_parse=True)
self.params = params
self.options = options
def run(self, comm, timing=False):
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
self.initialize()
self.validate(comm)
self.read_models()
timing = self.params.mpi.logging
if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt())
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt())
else:
if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt())
transmitted_info = None
if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt())
if timing: print "~BROADCAST START RANK=%d TIME=%f;" % (rank, tt())
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print "~BROADCAST END RANK=%d TIME=%f;" % (rank, tt())
# now actually do the work
if timing:
print "~SCALER_WORKER_SETUP START RANK=%d TIME=%f;" % (rank, tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print "~SCALER_WORKER_SETUP END RANK=%d TIME=%f;" % (rank, tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
# Use client-server distribution of work to the available MPI ranks.
# Each free rank requests a TAR ID and proceeds to process it.
if scaler_worker.params.mpi.cs == True:
tar_file_names = transmitted_info["file_names"]
if timing:
print "~SCALER_WORKERS START RANK=%d TIME=%f;" % (rank, tt())
if rank == 0:
for ix in range(len(tar_file_names)):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send(ix, dest=rankreq)
for rankreq in range(size - 1):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send('endrun', dest=rankreq)
scaler_worker.finished_db_mgr = db_mgr
else:
while True:
comm.send(rank, dest=0)
idx = comm.recv(source=0)
if idx == 'endrun':
scaler_worker.finished_db_mgr = db_mgr
break
if timing:
print "~SCALER_WORKER START=%d RANK=%d TIME=%f;" % (
idx, rank, tt())
scaler_worker._scale_all_serial([
tar_file_names[idx],
], db_mgr)
if timing:
print "~SCALER_WORKER END=%d RANK=%d TIME=%f;" % (
idx, rank, tt())
if timing:
print "~SCALER_WORKERS END RANK=%d TIME=%f;" % (rank, tt())
# Distribute chunks of TAR files to each MPI rank.
# The files are equidistributed across all the available ranks.
else:
file_names = [
transmitted_info["file_names"][i]
for i in range(len(transmitted_info["file_names"]))
if i % size == rank
]
if timing:
print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing:
print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
# gather reports and all add together
if timing: print "~GATHER START RANK=%d TIME=%f;" % (rank, tt())
reports = comm.gather(scaler_worker, root=0)
if timing: print "~GATHER END RANK=%d TIME=%f;" % (rank, tt())
if rank == 0:
print "Processing reports from %d ranks" % (len(reports))
ireport = 0
for item in reports:
if timing:
print "~SCALER_MASTER_ADD START RANK=%d TIME=%f;" % (rank,
tt())
scaler_master._add_all_frames(item)
if timing:
print "~SCALER_MASTER_ADD END RANK=%d TIME=%f;" % (rank,
tt())
print "processing %d calls from report %d" % (len(
item.finished_db_mgr.sequencer), ireport)
ireport += 1
for call_instance in item.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print "~SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f;" % (
rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print "~SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f;" % (
rank, tt())
elif call_instance["call"] == "insert_observation":
if timing:
print "~SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f;" % (
rank, tt())
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print "~SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f;" % (
rank, tt())
if timing:
print "~SCALER_MASTER_FINALISE START RANK=%d TIME=%f;" % (rank,
tt())
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print "~SCALER_MASTER_FINALISE END RANK=%d TIME=%f;" % (rank,
tt())
return self.finalize(scaler_master)
class Script(object):
'''A class for running the script.'''
def __init__(self, scaler_class):
# The script usage
import libtbx.load_env
self.usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name
self.parser = None
self.scaler_class = scaler_class
def initialize(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
from iotbx.phil import parse
phil_scope = parse(master_phil)
# Create the parser
self.parser = OptionParser(
usage=self.usage,
phil=phil_scope,
epilog=help_message)
self.parser.add_option(
'--plots',
action='store_true',
default=False,
dest='show_plots',
help='Show some plots.')
# Parse the command line. quick_parse is required for MPI compatibility
params, options = self.parser.parse_args(show_diff_phil=True,quick_parse=True)
self.params = params
self.options = options
def validate(self):
from xfel.merging.phil_validation import application
application(self.params)
if ((self.params.d_min is None) or
(self.params.data is None) or
( (self.params.model is None) and self.params.scaling.algorithm != "mark1") ) :
command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
raise Usage(command_name + " "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((self.params.rescale_with_average_cell) and
(not self.params.set_average_unit_cell)) :
raise Usage("If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
if [self.params.raw_data.sdfac_auto, self.params.raw_data.sdfac_refine, self.params.raw_data.errors_from_sample_residuals].count(True) > 1:
raise Usage("Specify only one of sdfac_auto, sdfac_refine or errors_from_sample_residuals.")
def read_models(self):
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s.log" % self.params.output.prefix, "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "I model"
if self.params.model is not None:
from xfel.merging.general_fcalc import run as run_fmodel
i_model = run_fmodel(self.params)
self.params.target_unit_cell = i_model.unit_cell()
self.params.target_space_group = i_model.space_group_info()
i_model.show_summary()
else:
i_model = None
print >> out, "Target unit cell and space group:"
print >> out, " ", self.params.target_unit_cell
print >> out, " ", self.params.target_space_group
from xfel.command_line.cxi_merge import consistent_set_and_model
self.miller_set, self.i_model = consistent_set_and_model(self.params,i_model)
self.frame_files = get_observations(self.params)
self.out = out
def scale_all(self):
scaler = self.scaler_class(
miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler.scale_all(self.frame_files)
return scaler
def finalize(self, scaler):
scaler.show_unit_cell_histograms()
if (self.params.rescale_with_average_cell) :
average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
average_cell = uctbx.unit_cell(list(average_cell_abc) +
list(self.params.target_unit_cell.parameters()[3:]))
self.params.target_unit_cell = average_cell
print >> out, ""
print >> out, "#" * 80
print >> out, "RESCALING WITH NEW TARGET CELL"
print >> out, " average cell: %g %g %g %g %g %g" % \
self.params.target_unit_cell.parameters()
print >> out, ""
scaler.reset()
scaler.scale_all(frame_files)
scaler.show_unit_cell_histograms()
if False : #(self.params.output.show_plots) :
try :
plot_overall_completeness(completeness)
except Exception as e :
print "ERROR: can't show plots"
print " %s" % str(e)
print >> self.out, "\n"
sum_I, sum_I_SIGI = scaler.sum_intensities()
miller_set_avg = self.miller_set.customized_copy(
unit_cell=self.params.target_unit_cell)
table1 = show_overall_observations(
obs=miller_set_avg,
redundancy=scaler.completeness,
redundancy_to_edge=scaler.completeness_predictions,
summed_wt_I=scaler.summed_wt_I,
summed_weight=scaler.summed_weight,
ISIGI=scaler.ISIGI,
n_bins=self.params.output.n_bins,
title="Statistics for all reflections",
out=self.out,
work_params=self.params)
print >> self.out, ""
if self.params.model is not None:
n_refl, corr = scaler.get_overall_correlation(sum_I)
else:
n_refl, corr = ((scaler.completeness > 0).count(True), 0)
print >> self.out, "\n"
table2 = show_overall_observations(
obs=miller_set_avg,
redundancy=scaler.summed_N,
redundancy_to_edge=scaler.completeness_predictions,
summed_wt_I=scaler.summed_wt_I,
summed_weight=scaler.summed_weight,
ISIGI=scaler.ISIGI,
n_bins=self.params.output.n_bins,
title="Statistics for reflections where I > 0",
out=self.out,
work_params=self.params)
#from libtbx import easy_pickle
#easy_pickle.dump(file_name="stats.pickle", obj=stats)
#stats.report(plot=self.params.plot)
#miller_counts = miller_set_p1.array(data=stats.counts.as_double()).select(
# stats.counts != 0)
#miller_counts.as_mtz_dataset(column_root_label="NOBS").mtz_object().write(
# file_name="nobs.mtz")
if self.params.data_subsubsets.subsubset is not None and self.params.data_subsubsets.subsubset_total is not None:
easy_pickle.dump("scaler_%d.pickle"%self.params.data_subsubsets.subsubset, scaler)
explanation = """
Explanation:
Completeness = # unique Miller indices present in data / # Miller indices theoretical in asymmetric unit
Asu. Multiplicity = # measurements / # Miller indices theoretical in asymmetric unit
Obs. Multiplicity = # measurements / # unique Miller indices present in data
Pred. Multiplicity = # predictions on all accepted images / # Miller indices theoretical in asymmetric unit"""
print >> self.out, explanation
mtz_file, miller_array = scaler.finalize_and_save_data()
#table_pickle_file = "%s_graphs.pkl" % self.params.output.prefix
#easy_pickle.dump(table_pickle_file, [table1, table2])
loggraph_file = os.path.abspath("%s_graphs.log" % self.params.output.prefix)
f = open(loggraph_file, "w")
f.write(table1.format_loggraph())
f.write("\n")
f.write(table2.format_loggraph())
f.close()
result = scaling_result(
miller_array=miller_array,
plots=scaler.get_plot_statistics(),
mtz_file=mtz_file,
loggraph_file=loggraph_file,
obs_table=table1,
all_obs_table=table2,
n_reflections=n_refl,
overall_correlation=corr)
easy_pickle.dump("%s.pkl" % self.params.output.prefix, result)
return result
def show_plot(self,result):
if result is not None:
if (self.options.show_plots) :
try :
result.plots.show_all_pyplot()
from wxtbx.command_line import loggraph
loggraph.run([result.loggraph_file])
except Exception as e :
print "Can't display plots"
print "You should be able to view them by running this command:"
print " wxtbx.loggraph %s" % result.loggraph_file
raise e
def run(self):
'''Execute the script.'''
self.initialize()
self.validate()
self.read_models()
scaler = self.scale_all()
result = self.finalize(scaler)
return result
class Script(object):
'''A class for running the script.'''
def __init__(self):
# The script usage
import libtbx.load_env
self.usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name
self.parser = None
def initialize(self):
'''Initialise the script.'''
from dials.util.options import OptionParser
from iotbx.phil import parse
phil_scope = parse(master_phil)
# Create the parser
self.parser = OptionParser(
usage=self.usage,
phil=phil_scope,
epilog=help_message)
self.parser.add_option(
'--plots',
action='store_true',
default=False,
dest='show_plots',
help='Show some plots.')
# Parse the command line. quick_parse is required for MPI compatibility
params, options = self.parser.parse_args(show_diff_phil=True,quick_parse=True)
self.params = params
self.options = options
def load_data(self):
from xfel.merging.application.input.file_loader import file_loader
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
# example showing what reading all the data into a single experiment list/
# reflection table would look like
loader = file_loader(self.params)
for experiments_filename, reflections_filename in loader.filepair_generator():
experiments, reflections = loader.load_data(experiments_filename, reflections_filename)
for experiment_id, experiment in enumerate(experiments):
all_experiments.append(experiment)
refls = reflections.select(reflections['id'] == experiment_id)
refls['id'] = flex.int(len(refls), len(all_experiments)-1)
all_reflections.extend(refls)
all_reflections.sort('miller_index_asymmetric')
return all_experiments, all_reflections
def validate(self):
from xfel.merging.application.validation.application import application
application(self.params)
def run(self):
print('''Mock run, merge some data.''')
self.initialize()
experiments, reflections = self.load_data()
print ('Read %d experiments consisting of %d reflections'%(len(experiments), len(reflections)))
self.validate()
# do other stuff
return