def onClusterHistogram(self, e):
if self.uc_values is not None:
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(info_list=[self.uc_values], legend_list=[])
plotter.plt.show()
def run(args):
user_phil = []
for arg in args:
try:
user_phil.append(parse(arg))
except Exception as e:
raise Sorry("Unrecognized argument %s" % arg)
params = phil_scope.fetch(sources=user_phil).extract()
app = xfel_db_application(params)
if params.tag is not None and len(params.tag) > 0:
tags = []
for tag in app.get_all_tags():
for t in params.tag:
if t == tag.name:
tags.append(tag)
extra_title = ",".join([t.name for t in tags])
else:
tags = None
extra_title = None
if params.run is not None and len(params.run) > 0:
assert params.rungroup is not None
runs = [app.get_run(run_number=r) for r in params.run]
rungroup = app.get_rungroup(rungroup_id=params.rungroup)
else:
runs = None
rungroup = None
if extra_title is None and runs is not None:
extra_title = "%s" % (",".join(["%s" % r.run for r in runs]))
trial = app.get_trial(trial_number=params.trial)
info = []
print("Reading data...")
cells = app.get_stats(trial=trial,
tags=tags,
isigi_cutoff=1.0,
tag_selection_mode=params.tag_selection_mode,
selected_runs=runs,
selected_rungroup=rungroup)()
for cell in cells:
info.append({
'a': cell.cell_a,
'b': cell.cell_b,
'c': cell.cell_c,
'alpha': cell.cell_alpha,
'beta': cell.cell_beta,
'gamma': cell.cell_gamma,
'n_img': 0
})
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(info_list=[info],
extra_title=extra_title,
legend_list=[""],
iqr_ratio=params.iqr_ratio)
plotter.plot_uc_3Dplot(info=info, iqr_ratio=params.iqr_ratio)
plotter.plt.show()
def run(self):
'''Execute the script.'''
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
if params.write_gnuplot_cloud:
gnuplot = open("uccloud.dat", 'w')
def get_info(experiment):
a, b, c, alpha, beta, gamma = experiment.crystal.get_unit_cell().parameters()
return {'a':a,
'b':b,
'c':c,
'alpha':alpha,
'beta':beta,
'gamma':gamma,
'n_img':0}
experiments_list = [e.data for e in params.input.experiments]
if params.extract_tags:
import os
experiments_tags = [os.path.basename(f.filename).split("_combined_experiments")[0] for f in params.input.experiments]
info_list = []
for experiments in experiments_list:
infos = []
for experiment in experiments:
info = get_info(experiment)
infos.append(info)
if params.write_gnuplot_cloud:
gnuplot.write("% 3.10f % 3.10f % 3.10f\n"%(info['a'], info['b'], info['c']))
info_list.append(infos)
else:
experiments_tags = [str(i) for i in range(len(experiments_list))]
info_list = []
for experiments in experiments_list:
infos = []
for experiment in experiments:
info = get_info(experiment)
infos.append(info)
if params.write_gnuplot_cloud:
gnuplot.write("% 3.10f % 3.10f % 3.10f\n"%(info['a'], info['b'], info['c']))
info_list.append(infos)
if params.combine_all_input:
info_list = [[info for infos in info_list for info in infos]]
experiments_tags = ["combined"]
if params.write_gnuplot_cloud:
gnuplot.close()
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(
info_list=info_list,
legend_list=experiments_tags,
iqr_ratio = params.iqr_ratio,
ranges = params.ranges,
title = params.title)
plotter.plt.show()
def run(self):
'''Execute the script.'''
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
def get_info(experiment):
a, b, c, alpha, beta, gamma = experiment.crystal.get_unit_cell(
).parameters()
return {
'a': a,
'b': b,
'c': c,
'alpha': alpha,
'beta': beta,
'gamma': gamma,
'n_img': 0
}
experiments_list = [e.data for e in params.input.experiments]
if params.extract_tags:
import os
experiments_tags = [
os.path.basename(f.filename).split("_combined_experiments")[0]
for f in params.input.experiments
]
info_list = []
for experiments in experiments_list:
infos = []
for experiment in experiments:
infos.append(get_info(experiment))
info_list.append(infos)
else:
experiments_tags = [""]
infos = []
for experiments in experiments_list:
for experiment in experiments:
infos.append(get_info(experiment))
info_list = [infos]
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(info_list=info_list,
legend_list=experiments_tags,
iqr_ratio=params.iqr_ratio,
ranges=params.ranges,
title=params.title)
plotter.plt.show()
def run(params, root, common_set=None):
iterable = []
iterable2 = []
debug_root = os.path.join(root, 'debug')
print('start appending')
for filename in os.listdir(debug_root):
if os.path.splitext(filename)[1] != ".txt": continue
iterable.append(filename)
for filename in os.listdir(root):
if 'refined_experiments' not in os.path.splitext(
filename)[0] or os.path.splitext(filename)[1] != ".json":
continue
iterable2.append(filename)
print('done appending')
#if command_line.options.mpi:
if params.mpi:
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
print(rank, size)
# get hits and indexing info
iterable = [
iterable[i] for i in range(len(iterable)) if (i + rank) % size == 0
]
results = get_hits_and_indexing_stats(iterable, debug_root, rank)
results = comm.gather(results, root=0)
# Now get uc and rmsd info
iterable2 = [
iterable2[i] for i in range(len(iterable2))
if (i + rank) % size == 0
]
results2 = get_uc_and_rmsd_stats(iterable2,
root,
rank=rank,
common_set=common_set)
results2 = comm.gather(results2, root=0)
if rank != 0: return
else:
results = [get_hits_and_indexing_stats(iterable, debug_root)]
results2 = [
get_uc_and_rmsd_stats(iterable2,
root,
rank=0,
common_set=common_set)
]
# Now evaulate summary statistics
print('Now evaluating summary statistics')
n_hits = 0
n_idx = 0
t_idx = 0
t_idx_success = 0
all_idx_cutoff_time_exceeded_event = []
total_xray_events = 0
total_images_analyzed = 0
for ii, r in enumerate(results):
n_hits += r[0]
n_idx += r[1]
t_idx += r[2]
t_idx_success += r[3]
all_idx_cutoff_time_exceeded_event.extend(r[4])
total_xray_events += r[5]
total_images_analyzed += r[6]
# Write out the cutoff time exceeded events in a format that xtc_process.py can interpret for skipping events
if indexing_time_cutoff is not None:
fts = open('timestamps_to_skip.dat', 'w')
for evt in all_idx_cutoff_time_exceeded_event:
fts.write('psanagpu999,%s,%s,fail\n' % (evt, evt))
fts.close()
node_hours = None
core_hours = None
if out_logfile is not None and wall_time is None:
total_time = []
run_number = int(os.path.abspath(root).strip().split('/')[-3][1:])
print(run_number)
with open(out_logfile, 'r') as flog:
for line in flog:
if string_to_search_for in line:
ax = line.split()
if int(ax[-1]) == run_number:
total_time.append(float(ax[1]))
node_hours = max(total_time) * num_nodes / 3600.0
if num_cores is not None:
core_hours = max(total_time) * num_cores / 3600.0
else:
core_hours = max(
total_time) * num_nodes * num_cores_per_node / 3600.0
if wall_time is not None:
node_hours = wall_time * num_nodes / 3600.0
if num_cores is not None:
core_hours = wall_time * num_cores / 3600.0
else:
core_hours = wall_time * num_nodes * num_cores_per_node / 3600.0
all_uc_a = flex.double()
all_uc_b = flex.double()
all_uc_c = flex.double()
all_uc_alpha = flex.double()
all_uc_beta = flex.double()
all_uc_gamma = flex.double()
dR = flex.double()
info_list = []
info = []
for ii, r in enumerate(results2):
all_uc_a.extend(r[0])
all_uc_b.extend(r[1])
all_uc_c.extend(r[2])
all_uc_alpha.extend(r[3])
all_uc_beta.extend(r[4])
all_uc_gamma.extend(r[5])
dR.extend(r[6])
if show_plot:
for jj, aa in enumerate(r[0]):
info.append({
'a': r[0][jj],
'b': r[1][jj],
'c': r[2][jj],
'alpha': r[3][jj],
'beta': r[4][jj],
'gamma': r[5][jj],
'n_img': 0
})
info_list.append(info)
n_lattices = len(all_uc_a)
# Now print out all relevant statistics
if True:
print('-' * 80)
print('|' + ' ' * 80 + '|\n' + '|' + ' ' * 20 +
'Analytics Package for Indexing' + ' ' * 30 + '|\n|' + ' ' * 80 +
'|')
print('-' * 80)
print('Getting stats for data in : ', root)
print(
'====================== Indexing and Timing Statistics ============================'
)
print('Total number of X-ray events = ', total_xray_events)
print('Total number of images analyzed = ', total_images_analyzed)
print('Number of Hits = ', n_hits)
print('Number of images successfully indexed = ', n_idx)
if common_set is None:
print('Number of lattices = ', n_lattices)
else:
print('Number of common lattices', n_lattices)
print('Total time spent in indexing (hrs) = ', t_idx)
print('Time spent in indexing successfully (core-hrs) = ',
t_idx_success)
print('Average time spent indexing (core-secs) = ',
3600 * t_idx / n_hits)
print('Average time spent indexing successfully (core-secs) = ',
3600 * t_idx_success / n_idx)
if node_hours is not None:
print('Total Node-hours with %d nodes = %.2f (hrs)' %
(num_nodes, node_hours))
print(
'% core utilization i.e (total indexing time)/(total core-hrs) = ',
100.0 * t_idx / core_hours)
if common_set is None:
print(
'====================== Unit Cell & RMSD Statistics ============================'
)
else:
print(
'====================== Unit Cell & RMSD Statistics from Common Set ============================'
)
print('a-edge (A) : %.2f +/- %.2f' %
(flex.mean(all_uc_a), flex.mean_and_variance(
all_uc_a).unweighted_sample_standard_deviation()))
print('b-edge (A) : %.2f +/- %.2f' %
(flex.mean(all_uc_b), flex.mean_and_variance(
all_uc_b).unweighted_sample_standard_deviation()))
print('c-edge (A) : %.2f +/- %.2f' %
(flex.mean(all_uc_c), flex.mean_and_variance(
all_uc_c).unweighted_sample_standard_deviation()))
print('alpha (deg) : %.2f +/- %.2f' %
(flex.mean(all_uc_alpha), flex.mean_and_variance(
all_uc_alpha).unweighted_sample_standard_deviation()))
print('beta (deg) : %.2f +/- %.2f' %
(flex.mean(all_uc_beta), flex.mean_and_variance(
all_uc_beta).unweighted_sample_standard_deviation()))
print('gamma (deg) : %.2f +/- %.2f' %
(flex.mean(all_uc_gamma), flex.mean_and_variance(
all_uc_gamma).unweighted_sample_standard_deviation()))
print('Total RMSD i.e calc - obs for Bragg spots (um) = ',
1000.0 * math.sqrt(dR.dot(dR) / len(dR)))
print('-' * 80)
if show_plot:
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(info_list=info_list,
legend_list=['combined'],
iqr_ratio=None)
plotter.plt.show()
info = []
print "Reading data..."
cells = app.get_stats(trial=trial,
tags=tags,
isigi_cutoff=1.0,
tag_selection_mode=params.tag_selection_mode,
selected_runs=runs,
selected_rungroup=rungroup)()
for cell in cells:
info.append({
'a': cell.cell_a,
'b': cell.cell_b,
'c': cell.cell_c,
'alpha': cell.cell_alpha,
'beta': cell.cell_beta,
'gamma': cell.cell_gamma,
'n_img': 0
})
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(info_list=[info],
extra_title=extra_title,
legend_list=[""],
iqr_ratio=params.iqr_ratio)
plotter.plot_uc_3Dplot(info=info, iqr_ratio=params.iqr_ratio)
plotter.plt.show()
if __name__ == "__main__":
run(sys.argv[1:])
def run(args):
user_phil = []
for arg in args:
try:
user_phil.append(parse(arg))
except Exception as e:
raise Sorry("Unrecognized argument %s" % arg)
params = phil_scope.fetch(sources=user_phil).extract()
app = xfel_db_application(params)
trial_ni = app.get_trial(trial_number=params.trial_noisoforms)
trial_i = app.get_trial(trial_number=params.trial_isoforms)
tags = []
if params.tag is not None and len(params.tag) > 0:
for tag in app.get_all_tags():
if params.tag == tag.name:
tags.append(tag)
extra_title = ",".join([t.name for t in tags])
tag_ids = [t.id for t in tags]
runs_ni = [
r for r in trial_ni.runs if any([t.id in tag_ids for t in r.tags])
]
runs_i = [
r for r in trial_i.runs if any([t.id in tag_ids for t in r.tags])
]
else:
extra_title = None
runs_ni = trial_ni.runs
runs_i = trial_i.runs
runs_ni_str = "(%s)" % (", ".join([str(r.id) for r in runs_ni]))
exp_tag = params.experiment_tag
isoforms = app.get_trial_isoforms(trial_i.id)
print "Reading data..."
info_list = []
legend_list = []
for isoform in isoforms:
events_i = app.get_all_events(trial=trial_i,
runs=runs_i,
isoform=isoform)
if len(events_i) == 0:
print "No events of isoform %s found" % isoform.name
continue
legend_list.append(isoform.name)
events_i_str = "(%s)" % (", ".join(
["'%s'" % e.timestamp for e in events_i]))
query = """SELECT crystal.cell_id FROM `%s_crystal` crystal
JOIN `%s_experiment` exp ON exp.crystal_id = crystal.id
JOIN `%s_imageset` imgset ON imgset.id = exp.imageset_id
JOIN `%s_imageset_event` ie ON ie.imageset_id = imgset.id
JOIN `%s_event` evt ON evt.id = ie.event_id
JOIN `%s_trial` trial ON trial.id = evt.trial_id
JOIN `%s_run` run ON run.id = evt.run_id
WHERE run.id in %s AND trial.id = %d AND evt.timestamp IN %s""" % (
exp_tag, exp_tag, exp_tag, exp_tag, exp_tag, exp_tag, exp_tag,
runs_ni_str, trial_ni.id, events_i_str)
cell_ids = [str(i[0]) for i in app.execute_query(query).fetchall()]
if len(cell_ids) == 0:
cells = []
else:
from xfel.ui.db.experiment import Cell
cells = app.get_all_x(Cell,
'cell',
where="WHERE id IN (%s)" %
", ".join(cell_ids))
print len(cells)
info = []
for cell in cells:
info.append({
'a': cell.cell_a,
'b': cell.cell_b,
'c': cell.cell_c,
'alpha': cell.cell_alpha,
'beta': cell.cell_beta,
'gamma': cell.cell_gamma,
'n_img': 0
})
info_list.append(info)
import xfel.ui.components.xfel_gui_plotter as pltr
plotter = pltr.PopUpCharts()
plotter.plot_uc_histogram(info_list=info_list,
legend_list=legend_list,
extra_title=extra_title)
plotter.plt.show()
