def run(args):
import os
to_pickle = "--pickle" in args
for file_name in args:
if (file_name.startswith("--")): continue
print file_name + ":"
f = open(file_name, "r")
t0 = os.times()
reflection_file = cns_reflection_file(f)
tn = os.times()
t_parse = tn[0]+tn[1]-t0[0]-t0[1]
f.close()
reflection_file.show_summary()
print
crystal_symmetry = crystal.symmetry((), "P 1")
miller_arrays = reflection_file.as_miller_arrays(crystal_symmetry)
for miller_array in miller_arrays:
miller_array.show_summary()
print
if (to_pickle):
pickle_file_name = os.path.split(file_name)[1] + ".pickle"
t0 = os.times()
easy_pickle.dump(pickle_file_name, reflection_file)
tn = os.times()
t_dump = tn[0]+tn[1]-t0[0]-t0[1]
t0 = os.times()
easy_pickle.load(pickle_file_name)
tn = os.times()
t_load = tn[0]+tn[1]-t0[0]-t0[1]
print "parse: %.2f, dump: %.2f, load: %.2f" % (t_parse, t_dump, t_load)
print
t = os.times()
print "u+s,u,s: %.2f %.2f %.2f" % (t[0] + t[1], t[0], t[1])
def __init__(self,
init,
iterable,
input_type='image'):
self.init = ep.load(init)
self.iterable = ep.load(iterable)
self.type = input_type
def __init__ (self, dir_name) :
self.dir_name = dir_name
self._distl = None
self._labelit = None
self._groups = None
distl_file = os.path.join(dir_name, "DISTL_pickle")
labelit_file = os.path.join(dir_name, "LABELIT_pickle")
groups_file = os.path.join(dir_name, "LABELIT_possible")
if (os.path.exists(distl_file)) :
self._distl = easy_pickle.load(distl_file)
if (os.path.exists(labelit_file)) :
self._labelit = easy_pickle.load(labelit_file)
if (os.path.exists(groups_file)) :
self._groups = easy_pickle.load(groups_file)
def analyze_prior_results(self, analysis_source):
""" Runs analysis of previous grid search / integration results, used in an
analyze-only mode """
from prime.iota.iota_analysis import Analyzer
from libtbx import easy_pickle as ep
if os.path.isdir(analysis_source):
int_folder = os.path.abspath(analysis_source)
else:
try:
int_folder = os.path.abspath(os.path.join(os.curdir,
'integration/{}/image_objects'.format(analysis_source)))
except ValueError:
print 'Run #{} not found'.format(analysis_source)
if os.path.isdir(int_folder):
int_list = [os.path.join(int_folder, i) for i in os.listdir(int_folder)]
img_objects = [ep.load(i) for i in int_list]
analysis = Analyzer(img_objects, None, self.iver, self.now)
analysis.print_results()
analysis.unit_cell_analysis(self.params.analysis.cluster_threshold,
int_folder, False)
analysis.print_summary(None)
analysis.show_heatmap()
else:
print 'No results found in {}'.format(int_folder)
def exercise_02():
file = libtbx.env.find_in_repositories(relative_path=
"chem_data/polygon_data/all_mvd.pickle", test=os.path.isfile)
database_dict = easy_pickle.load(file)
#
r_work_pdb = database_dict["pdb_header_r_work"]
r_work_cutoff = database_dict["r_work_cutoffs"]
r_work_re_computed = database_dict["r_work"]
name = database_dict["pdb_code"]
print "size: ",name.size()
#
sel = r_work_pdb != "none"
print "NO R in PDB:", sel.count(False)
#
r_work_pdb = r_work_pdb.select(sel)
r_work_cutoff = r_work_cutoff.select(sel)
r_work_re_computed = r_work_re_computed.select(sel)
name = name.select(sel)
#
def str_to_float(x):
tmp = flex.double()
for x_ in x:
tmp.append(float(x_))
return tmp
#
r_work_cutoff = str_to_float(r_work_cutoff)
r_work_re_computed = str_to_float(r_work_re_computed)
r_work_pdb = str_to_float(r_work_pdb)
#
delta1 = (r_work_cutoff - r_work_pdb)*100.
delta2 = (r_work_re_computed - r_work_pdb)*100.
print "Number with better R: ", (delta1 <=0.).count(True), (delta2 <=0.).count(True)
print "Number with worse R: ", (delta1 >0.).count(True), (delta2 >0.).count(True)
def __init__(self, pickle, img_location, pixel_size, proceed_without_image=False):
# unpickle pickle file and keep track of image location
if img_location is None:
if proceed_without_image:
self.img_location = []
else:
raise Sorry, "No image found at specified location. Override by setting proceed_without_image to False" \
+ "to produce experiment lists that may only be read when check_format is False."
else:
self.img_location = [img_location]
# pickle can be already unpickled, if so, loading it will fail with an AttributeError. A load
# error will fail with EOFError
try:
self.data = easy_pickle.load(pickle)
except EOFError:
self.data = None
self.pickle = None
except AttributeError:
self.data = pickle
self.pickle = None
else:
self.pickle = pickle
if self.data is not None:
self.length = len(self.data['observations'][0].data())
self.pixel_size = pixel_size
def _get_classifier(svm_name=None):
"""
If need be, initializes, and then returns a classifier trained to
differentiate between different ions and water. Also returns of options for
gathering features.
To use the classifier, you will need to pass it to
svm.libsvm.svm_predict_probability. Ion prediction is already encapsulated by
predict_ion, so most users should just call that.
Parameters
----------
svm_name : str, optional
The SVM to use for prediction. By default, the SVM trained on heavy atoms
and calcium in the presence of anomalous data is used. See
chem_data/classifiers for a full list of SVMs available.
Returns
-------
svm.svm_model
The libsvm classifier used to predict the identities of ion sites.
dict of str, bool
Options to pass to ion_vector when collecting features about these sites.
tuple of ((tuple of numpy.array of float, numpy.array of float),
tuple of float)
The scaling parameters passed to scale_to.
numpy.array of bool
The features of the vector that were selected as important for
classification. Useful for both asserting that ion_vector is returning
something of the correct size as well as only selection features that
actually affect classification.
See Also
--------
svm.libsvm.svm_predict_probability
mmtbx.ions.svm.predict_ion
phenix_dev.ion_identification.nader_ml.ions_train_svms
"""
assert (svmutil is not None)
global _CLASSIFIER, _CLASSIFIER_OPTIONS
if not svm_name or str(svm_name) == "Auto" :
svm_name = _DEFAULT_SVM_NAME
if svm_name not in _CLASSIFIER:
svm_path = os.path.join(CLASSIFIERS_PATH, "{}.model".format(svm_name))
options_path = os.path.join(CLASSIFIERS_PATH,
"{}_options.pkl".format(svm_name))
try:
_CLASSIFIER[svm_name] = svmutil.svm_load_model(svm_path)
except IOError as err:
if err.errno != errno.ENOENT:
raise err
else:
_CLASSIFIER[svm_name] = None
_CLASSIFIER_OPTIONS[svm_name] = (None, None, None)
_CLASSIFIER_OPTIONS[svm_name] = load(options_path)
vector_options, scaling, features = _CLASSIFIER_OPTIONS[svm_name]
return _CLASSIFIER[svm_name], vector_options, scaling, features
def exercise_00():
file = libtbx.env.find_in_repositories(relative_path=
"chem_data/polygon_data/all_mvd.pickle", test=os.path.isfile)
database_dict = easy_pickle.load(file)
#
i_1l2h = list(database_dict["pdb_code"]).index("1l2h")
found = 0
for key in database_dict.keys():
#print key, ": ",database_dict[key][i_1l2h]
#
value = database_dict[key][i_1l2h]
if(key == "unit_cell"):
assert "(53.89, 53.89, 77.36, 90, 90, 90)" == value.strip()
found += 1
if(key == "high_resolution"):
assert approx_equal(1.54, float(value))
found += 1
if(key == "pdb_header_tls"):
assert "false (n_groups: 0)" == value.strip()
found += 1
if(key == "test_set_size"):
assert approx_equal(0.0476, float(value),0002)
found += 1
if(key == "twinned"):
assert "h,-k,-l" == value.strip()
found += 1
#
assert found == 5
def __init__(self, scratch_dir):
pickle_dirname = "pickle"
pickle_basename = "pkl_"
self.sum_img = None
self.sumsq_img = None
self.nmemb = 0
path_pattern = "%s/%s/%ss[0-9][0-9]-[0-9].pickle" %(
scratch_dir, pickle_dirname, pickle_basename)
g = glob.glob(path_pattern)
if len(g) == 0:
print "No files found matching pattern: %s" %path_pattern
return
for path in g:
try:
d = easy_pickle.load(file_name=path)
except EOFError:
print "EOFError: skipping %s:" %path
continue
if self.sum_img is None:
self.sum_img = d["sum_img"]
self.sumsq_img = d["sumsq_img"]
else:
self.sum_img += d["sum_img"]
self.sumsq_img += d["sumsq_img"]
self.nmemb += d["nmemb"]
print "Read %d images from %s" % (d["nmemb"], path)
#print "Si foil length: %s" %(d["sifoil"])
print "Number of images used: %i" %self.nmemb
assert self.nmemb > 0
def analyze_prior_results(self, analysis_source):
""" Runs analysis of previous grid search / integration results, used in an
analyze-only mode """
from iota.components.iota_analysis import Analyzer
from libtbx import easy_pickle as ep
if os.path.isdir(analysis_source):
int_folder = os.path.abspath(analysis_source)
else:
try:
int_folder = os.path.abspath(os.path.join(os.curdir,
'integration/{}/image_objects'.format(analysis_source)))
except ValueError:
print 'Run #{} not found'.format(analysis_source)
if os.path.isdir(int_folder):
int_list = [os.path.join(int_folder, i) for i in os.listdir(int_folder)]
img_objects = [ep.load(i) for i in int_list if i.endswith('.int')]
self.logfile = os.path.abspath(os.path.join(int_folder, 'iota.log'))
analysis = Analyzer(self, img_objects, self.iver)
analysis.print_results()
analysis.unit_cell_analysis(write_files=False)
analysis.print_summary(write_files=False)
else:
print 'No results found in {}'.format(int_folder)
def OnChangeSymmetry (event) :
from mmtbx.scaling.xtriage import change_symmetry
button = event.GetEventObject()
data_file = button.data_file
assert (data_file is not None) and os.path.isfile(data_file)
item = button.symm_table.GetFirstSelected()
if (item == -1) :
raise Sorry("Please select a symmetry setting first!")
default_file = os.path.join(os.path.dirname(data_file), "reindexed.mtz")
file_name = wxtbx.path_dialogs.manager().select_file(
parent=button,
message="Output data file in new symmetry",
style=wx.SAVE,
wildcard="MTZ files (*.mtz)|*.mtz",
current_file=default_file)
data = easy_pickle.load(data_file)
space_group_symbol = str(button.symm_table.GetItemText(item))
change_symmetry(
miller_array=data,
space_group_symbol=space_group_symbol,
file_name=str(file_name),
log=sys.stdout)
assert os.path.isfile(file_name)
wx.MessageBox(("The data (as amplitudes) have been saved to %s "+
"with the selected symmetry setting.") % file_name)
def get_rotarama_data (residue_type=None, pos_type=None, db="rama",
convert_to_numpy_array=False) :
from mmtbx.rotamer import ramachandran_eval
from mmtbx.rotamer.rotamer_eval import find_rotarama_data_dir
# backwards compatibility
if (pos_type == "proline") : pos_type = "trans-proline"
if (pos_type == "prepro") : pos_type = "pre-proline"
assert (pos_type in ["general", "cis-proline", "trans-proline", "glycine",
"isoleucine or valine", "pre-proline",None])
assert (db in ["rama", "rota"])
assert (residue_type is not None) or (pos_type is not None)
if pos_type is not None :
residue_type = ramachandran_eval.aminoAcids_8000[pos_type]
if residue_type.lower() in ["phe", "tyr"] :
residue_type = "phetyr"
assert (residue_type is not None)
rama_data_dir = find_rotarama_data_dir()
if (db == "rama") :
pkl_file = "%s8000-%s.pickle" % (db, residue_type)
else :
pkl_file = "%s8000-%s.pickle" % (db, residue_type.lower())
ndt = easy_pickle.load(os.path.join(rama_data_dir, pkl_file))
if convert_to_numpy_array :
if (db == "rama") :
return export_ramachandran_distribution(ndt)
else :
return export_rotamer_distribution(ndt)
else :
return ndt
def __init__(self, params): self.src = psana.Source(params.spectra_filter.detector_address) self.roi = params.spectra_filter.roi self.bg_roi = params.spectra_filter.background_roi self.dark_pickle = easy_pickle.load(params.spectra_filter.background_path)['DATA'].as_numpy_array() self.peak_range = params.spectra_filter.peak_range self.params = params
def run(args):
assert len(args) == 3
d1 = easy_pickle.load(args[0])
d2 = easy_pickle.load(args[1])
image_1 = d1["DATA"]
image_2 = d2["DATA"]
assert image_1.all() == image_2.all()
diff_image = image_1 - image_2
d = cspad_tbx.dpack(
data=diff_image,
timestamp=cspad_tbx.evt_timestamp(),
distance=1,
)
easy_pickle.dump(args[2], d)
def __init__(self):
main_aaTables = RamachandranEval.aaTables
self.aaTables = {}
for aa,ndt_weakref in main_aaTables.items():
# convert existing weak references to strong references
self.aaTables[aa] = ndt_weakref()
rama_data_dir = find_rotarama_data_dir()
target_db = open_rotarama_dlite(rotarama_data_dir=rama_data_dir)
no_update = os.path.exists(os.path.join(rama_data_dir, "NO_UPDATE"))
for aa, aafile in aminoAcids_8000.items():
if (self.aaTables.get(aa) is not None): continue
data_file = "rama8000-"+aafile+".data"
pickle_file = "rama8000-"+aafile+".pickle"
pair_info = target_db.pair_info(
source_path=data_file,
target_path=pickle_file,
path_prefix=rama_data_dir)
if (((pair_info.needs_update) and (not no_update)) or not
os.path.exists(os.path.join(rama_data_dir, pickle_file))) :
raise Sorry(
"chem_data/rotarama_data/*.pickle files are missing or out of date.\n"
" Please run\n"
" mmtbx.rebuild_rotarama_cache\n"
" to resolve this problem.\n")
ndt = easy_pickle.load(file_name=os.path.join(
rama_data_dir, pair_info.target.path))
self.aaTables[aa] = ndt
main_aaTables[aa] = weakref.ref(ndt)
def get_mean_statistic_for_resolution (d_min, stat_type, range=0.2, out=None) :
if (out is None) :
out = sys.stdout
from scitbx.array_family import flex
pkl_file = libtbx.env.find_in_repositories(
relative_path = "chem_data/polygon_data/all_mvd.pickle",
test = os.path.isfile)
db = easy_pickle.load(pkl_file)
all_d_min = db['high_resolution']
stat_values = db[stat_type]
values_for_range = flex.double()
for (d_, v_) in zip(all_d_min, stat_values) :
try :
d = float(d_)
v = float(v_)
except ValueError : continue
else :
if (d > (d_min - range)) and (d < (d_min + range)) :
values_for_range.append(v)
h = flex.histogram(values_for_range, n_slots=10)
print >> out, " %s for d_min = %.3f - %.3f A" % (stat_names[stat_type], d_min-range,
d_min+range)
min = flex.min(values_for_range)
max = flex.max(values_for_range)
mean = flex.mean(values_for_range)
print >> out, " count: %d" % values_for_range.size()
print >> out, " min: %.2f" % min
print >> out, " max: %.2f" % max
print >> out, " mean: %.2f" % mean
print >> out, " histogram of values:"
h.show(prefix=" ")
return mean
def make_int_object_list(self):
""" Generates list of image objects from previous grid search """
from libtbx import easy_pickle as ep
if self.params.cctbx.selection.select_only.grid_search_path == None:
int_dir = misc.set_base_dir('integration', True)
else:
int_dir = self.params.cctbx.selection.select_only.grid_search_path
img_objects = []
cmd.Command.start("Importing saved grid search results")
for root, dirs, files in os.walk(int_dir):
for filename in files:
found_file = os.path.join(root, filename)
if found_file.endswith(('int')):
obj = ep.load(found_file)
img_objects.append(obj)
cmd.Command.end("Importing saved grid search results -- DONE")
# Pick a randomized subset of images
if self.params.advanced.random_sample.flag_on and \
self.params.advanced.random_sample.number < len(img_objects):
gs_img_objects = self.select_random_subset(img_objects)
else:
gs_img_objects = img_objects
return gs_img_objects
def run(args):
if len(args)==0: print Usage; exit()
processed = iotbx.phil.process_command_line(
args=args, master_string=master_phil_str)
args = processed.remaining_args
work_params = processed.work.extract().xes
processed.work.show()
assert len(args) == 1
output_dirname = work_params.output_dirname
roi = cspad_tbx.getOptROI(work_params.roi)
bg_roi = cspad_tbx.getOptROI(work_params.bg_roi)
gain_map_path = work_params.gain_map
estimated_gain = work_params.estimated_gain
print output_dirname
if output_dirname is None:
output_dirname = os.path.join(os.path.dirname(args[0]), "finalise")
print output_dirname
hist_d = easy_pickle.load(args[0])
if len(hist_d.keys())==2:
hist_d = hist_d['histogram']
pixel_histograms = faster_methods_for_pixel_histograms(
hist_d, work_params)
result = xes_from_histograms(
pixel_histograms, output_dirname=output_dirname,
gain_map_path=gain_map_path, estimated_gain=estimated_gain,
roi=roi, run=work_params.run)
def __init__(self, scratch_dir, pickle_pattern=None):
pickle_dirname = "pickle"
pickle_basename = "pkl_"
self.nmemb = 0
self.histogram = None
if pickle_pattern is not None:
path_pattern = "%s/%s/%s" %(scratch_dir, pickle_dirname, pickle_pattern)
else:
path_pattern = "%s/%s/%ss[0-9][0-9]-*.pickle" %(
scratch_dir, pickle_dirname, pickle_basename)
print path_pattern
g = glob.glob(path_pattern)
if len(g) == 0:
print "No matches found for pattern: %s" %path_pattern
return
for path in g:
try:
d = easy_pickle.load(file_name=path)
except EOFError:
print "EOFError: skipping %s:" %path
continue
if len(d["histogram"].keys()) == 0: continue
if self.histogram is None:
self.histogram = d["histogram"]
else:
self.histogram = update_histograms(self.histogram, d["histogram"])
self.nmemb += d["nmemb"]
print "Read %d images from %s" % (d["nmemb"], path)
print "Number of images used: %i" %self.nmemb
def __init__(self,address,
peak_one_position_min,
peak_one_position_max,
peak_two_position_min,
peak_two_position_max,
peak_one_width,
peak_two_width,
peak_ratio = 0.4,
normalized_peak_to_noise_ratio = 0.4,
spectrometer_dark_path = None,
iron_edge_position = None):
"""The mod_spectrum_filter class constructor stores the parameters passed
from the psana configuration file in instance variables.
@param address Full data source address of the FEE device
@param peak_one_position_min the minimum x coordinate in pixel units
of the first peak position on the detector
@param peak_one_position_max the maximum x coordinate in pixel units
of the first peak position on the detector
@param peak_two_position_min the minimum x coordinate in pixel units
of the second peak position on the detector
@param peak_two_position_max the maximum x coordinate in pixel units
of the second peak position on the detector
@param peak_one_width the width in pixels of the first peak
@param peak_two_width the width in pixels of the second peak
@param peak_ratio the ratio of the two peak heights
@param normalized_peak_to_noise_ratio ratio of the normalized integrated
peak to the normalized integrated regions between the peaks
@param spectrometer_dark_path path to pickle file of FEE dark
if None then no dark is subtracted from the spectrum
@param iron_edge_position the position in pixels of the iron edge if absorbing
iron foil is used in the experiment if None this is not used as a filtering parameter
"""
#self.logger = logging.getLogger(self.__class__.__name__)
#self.logger.setLevel(logging.INFO)
#self.logging = logging
self.src = Source('%s'%address)
if spectrometer_dark_path is not None:
self.dark = easy_pickle.load(spectrometer_dark_path)
else:
self.dark = None
self.peak_one_position_min = int(peak_one_position_min)
self.peak_one_position_max = int(peak_one_position_max)
self.peak_two_position_min = int(peak_two_position_min)
self.peak_two_position_max = int(peak_two_position_max)
self.peak_one_width = int(peak_one_width)
self.peak_two_width = int(peak_two_width)
self.normalized_peak_to_noise_ratio = float(normalized_peak_to_noise_ratio)
self.peak_ratio = float(peak_ratio)
if iron_edge_position is not None:
self.iron_edge_position = int(iron_edge_position)
else:
self.iron_edge_position = None
self.ntwo_color = 0
self.nnodata = 0
self.nshots = 0
self.naccepted= 0
def run(args):
for file_name in args:
structures = easy_pickle.load(file_name)
if (isinstance(structures, xray.structure)):
structures = [structures]
for structure in structures:
structure.show_summary().show_scatterers()
print
def load_db (file_name=None) :
if(file_name is None):
file_name = libtbx.env.find_in_repositories(
relative_path = "chem_data/polygon_data/all_mvd.pickle",
test = os.path.isfile)
assert os.path.isfile(file_name)
database_dict = easy_pickle.load(file_name)
return database_dict
def run(files, gain, prefix):
from libtbx import easy_pickle
for file in files:
f = easy_pickle.load(file)
old_miller = f['observations'][0]
new_miller = old_miller.customized_copy(sigmas=gain * old_miller.sigmas())
f['observations'][0] = new_miller
easy_pickle.dump(prefix + file, f)
def unpickle_miller_arrays(file_name):
result = easy_pickle.load(file_name)
if (isinstance(result, miller.array)):
return [result]
result = list(result)
for miller_array in result:
if (not isinstance(miller_array, miller.array)):
return None
return result
def run(args):
processed = iotbx.phil.process_command_line(
args=args, master_string=master_phil_str)
args = processed.remaining_args
work_params = processed.work.extract().xes
processed.work.show()
assert len(args) == 1
output_dirname = work_params.output_dirname
roi = cspad_tbx.getOptROI(work_params.roi)
bg_roi = cspad_tbx.getOptROI(work_params.bg_roi)
gain_map_path = work_params.gain_map
estimated_gain = work_params.estimated_gain
nproc = work_params.nproc
photon_threshold = work_params.photon_threshold
method = work_params.method
print output_dirname
if output_dirname is None:
output_dirname = os.path.join(os.path.dirname(args[0]), "finalise")
print output_dirname
hist_d = easy_pickle.load(args[0])
if len(hist_d.keys())==2:
hist_d = hist_d['histogram']
pixel_histograms = view_pixel_histograms.pixel_histograms(
hist_d, estimated_gain=estimated_gain)
result = xes_from_histograms(
pixel_histograms, output_dirname=output_dirname,
gain_map_path=gain_map_path, estimated_gain=estimated_gain,
method=method, nproc=nproc,
photon_threshold=photon_threshold, roi=roi, run=work_params.run)
if bg_roi is not None:
bg_outdir = os.path.normpath(output_dirname)+"_bg"
bg_result = xes_from_histograms(
pixel_histograms, output_dirname=bg_outdir,
gain_map_path=gain_map_path, estimated_gain=estimated_gain,
method=method, nproc=nproc,
photon_threshold=photon_threshold, roi=bg_roi)
from xfel.command_line.subtract_background import subtract_background
signal = result.spectrum
background = bg_result.spectrum
signal = (signal[0].as_double(), signal[1])
background = (background[0].as_double(), background[1])
signal_x, background_subtracted = subtract_background(signal, background, plot=True)
f = open(os.path.join(output_dirname, "background_subtracted.txt"), "wb")
print >> f, "\n".join(["%i %f" %(x, y)
for x, y in zip(signal_x, background_subtracted)])
f.close()
else:
from xfel.command_line import smooth_spectrum
from scitbx.smoothing import savitzky_golay_filter
x, y = result.spectrum[0].as_double(), result.spectrum[1]
x, y = smooth_spectrum.interpolate(x, y)
x, y_smoothed = savitzky_golay_filter(
x, y, 20, 4)
smooth_spectrum.estimate_signal_to_noise(x, y, y_smoothed)
def get_image_integration (self, sol_id, image_id=None, file_name=None) :
assert (image_id is not None) or (file_name is not None)
file_name = os.path.join(self.dir_name, "integration_%d_%d.pkl" % (sol_id,
image_id))
if (not os.path.exists(file_name)) :
raise Sorry("Can't find the file %s!" % file_name)
integ_result = easy_pickle.load(file_name)
summary = get_integration_summary(integ_result, image_id)
return integ_result, summary
def __init__(self, file_name, file_name_during_write=None):
self.file_name = file_name
if (file_name_during_write is None and self.file_name is not None):
self.file_name_during_write = self.file_name + ".new"
else:
self.file_name_during_write = file_name_during_write
if (self.file_name is None
or not os.path.exists(self.file_name)):
self.pair_infos = {}
else:
self.pair_infos = easy_pickle.load(file_name=self.file_name)
def plot_runtime_results(self):
''' Plot results for each cycle upon cycle completion '''
stat_file = os.path.join(self.pparams.run_no, 'pickle.stat')
if os.path.isfile(stat_file):
info = ep.load(stat_file)
else:
info = {}
total_cycles = self.pparams.n_postref_cycle
if 'binned_resolution' in info:
self.graph_tab.draw_plots(info, total_cycles)
def readHeader(self):
# XXX The functionality provided by this function has largely been
# replicated in
# rstbx.slip_viewer.tile_generation._get_flex_image_multitile().
# However, several code paths still depend on the member variables
# created here.
from xfel.cftbx.detector.metrology import metrology_as_transformation_matrices
d = easy_pickle.load(self.filename)
# Derive the transformation matrices from the metrology in the
# image.
self._metrology_params = d["METROLOGY"].extract()
self._matrices = metrology_as_transformation_matrices(
self._metrology_params)
self._tiles = d["TILES"]
self._keylist = self._tiles.keys() #for later use by get_pixel_intensity()
# Assert that all ASIC:s are the same size, and that there are
# transformation matrices for each ASIC.
for (key, asic) in self._tiles.iteritems():
if not hasattr(self, "_asic_focus"):
self._asic_focus = asic.focus()
else:
assert asic.focus() == self._asic_focus
assert key in self._matrices
# Assert that all specified pixel sizes and saturated values are
# equal and not None.
for p in self._metrology_params.detector.panel:
for s in p.sensor:
for a in s.asic:
if not hasattr(self, "_pixel_size"):
self._pixel_size = a.pixel_size
else:
assert self._pixel_size == a.pixel_size
if not hasattr(self, "_saturation"):
self._saturation = a.saturation
else:
# XXX real-valued equality! See
# cctbx_project/scitbx/math/approx_equal.h
assert self._saturation == a.saturation
assert hasattr(self, "_pixel_size") and self._pixel_size is not None
assert hasattr(self, "_saturation") and self._saturation is not None
# Determine next multiple of eight. Set size1 and size2 to the
# focus of the padded rawdata.
self._asic_padded = (8 * int(math.ceil(self._asic_focus[0] / 8)),
8 * int(math.ceil(self._asic_focus[1] / 8)))
self.size1 = len(self._tiles) * self._asic_padded[0]
self.size2 = self._asic_padded[1]
def fetch_fingerprints(motif_list):
fingerprint_list = []
for motif in motif_list:
path = "cctbx_project/mmtbx/cablam/fingerprints/"+motif+".pickle"
picklefile = libtbx.env.find_in_repositories(
relative_path=path, test=os.path.isfile)
if picklefile is None:
raise Sorry("\nCould not find a needed pickle file for motif "+motif+" in chem_data.\nExiting.\n")
else:
fingerprint_list.append(easy_pickle.load(file_name=picklefile))
return fingerprint_list
print picklepath
destpath_l = os.path.join(
destroot_l,
os.path.splitext(picklename)[0] + "_l.pickle")
destpath_r = os.path.join(
destroot_r,
os.path.splitext(picklename)[0] + "_r.pickle")
destpath_m = os.path.join(
destroot_m,
os.path.splitext(picklename)[0] + "_m.pickle")
destpath_n = os.path.join(
destroot_n,
os.path.splitext(picklename)[0] + "_n.pickle")
#if os.path.exists(destpath_l): continue
try:
data = easy_pickle.load(picklepath)
except Exception as e:
print "Pickle failed to load", picklepath
continue
if not "fuller_kapton_absorption_correction" in data:
continue
corr = data["fuller_kapton_absorption_correction"]
from xfel.cxi.cspad_ana.rayonix_tbx import get_rayonix_pixel_size
from scitbx.array_family import flex
pixel_size = get_rayonix_pixel_size(2)
bx = data['xbeam'] / pixel_size
by = data['ybeam'] / pixel_size
preds = data['mapped_predictions']
def run(args):
processed = iotbx.phil.process_command_line(args=args,
master_string=master_phil_str)
args = processed.remaining_args
work_params = processed.work.extract().xes
processed.work.show()
assert len(args) == 1
output_dirname = work_params.output_dirname
roi = cspad_tbx.getOptROI(work_params.roi)
bg_roi = cspad_tbx.getOptROI(work_params.bg_roi)
gain_map_path = work_params.gain_map
estimated_gain = work_params.estimated_gain
nproc = work_params.nproc
photon_threshold = work_params.photon_threshold
method = work_params.method
print(output_dirname)
if output_dirname is None:
output_dirname = os.path.join(os.path.dirname(args[0]), "finalise")
print(output_dirname)
hist_d = easy_pickle.load(args[0])
if len(hist_d) == 2:
hist_d = hist_d['histogram']
pixel_histograms = view_pixel_histograms.pixel_histograms(
hist_d, estimated_gain=estimated_gain)
result = xes_from_histograms(pixel_histograms,
output_dirname=output_dirname,
gain_map_path=gain_map_path,
estimated_gain=estimated_gain,
method=method,
nproc=nproc,
photon_threshold=photon_threshold,
roi=roi,
run=work_params.run)
if bg_roi is not None:
bg_outdir = os.path.normpath(output_dirname) + "_bg"
bg_result = xes_from_histograms(pixel_histograms,
output_dirname=bg_outdir,
gain_map_path=gain_map_path,
estimated_gain=estimated_gain,
method=method,
nproc=nproc,
photon_threshold=photon_threshold,
roi=bg_roi)
from xfel.command_line.subtract_background import subtract_background
signal = result.spectrum
background = bg_result.spectrum
signal = (signal[0].as_double(), signal[1])
background = (background[0].as_double(), background[1])
signal_x, background_subtracted = subtract_background(signal,
background,
plot=True)
f = open(os.path.join(output_dirname, "background_subtracted.txt"),
"wb")
print("\n".join([
"%i %f" % (x, y) for x, y in zip(signal_x, background_subtracted)
]),
file=f)
f.close()
else:
from xfel.command_line import smooth_spectrum
from scitbx.smoothing import savitzky_golay_filter
x, y = result.spectrum[0].as_double(), result.spectrum[1]
x, y = smooth_spectrum.interpolate(x, y)
x, y_smoothed = savitzky_golay_filter(x, y, 20, 4)
smooth_spectrum.estimate_signal_to_noise(x, y, y_smoothed)
def run (args, imageset = None):
from xfel import radial_average
from scitbx.array_family import flex
import os, sys
from dxtbx.datablock import DataBlockFactory
# Parse input
try:
n = len(args)
except Exception:
params = args
else:
user_phil = []
for arg in args:
if (not "=" in arg):
try :
user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg))
except ValueError:
raise Sorry("Unrecognized argument '%s'" % arg)
else:
try:
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError as e:
raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if imageset is None:
if params.file_path is None or len(params.file_path) == 0 or not all([os.path.isfile(f) for f in params.file_path]):
master_phil.show()
raise Usage("file_path must be defined (either file_path=XXX, or the path alone).")
assert params.n_bins is not None
assert params.verbose is not None
assert params.output_bins is not None
# Allow writing to a file instead of stdout
if params.output_file is None:
logger = sys.stdout
else:
logger = open(params.output_file, 'w')
logger.write("%s "%params.output_file)
if params.show_plots:
from matplotlib import pyplot as plt
import numpy as np
colormap = plt.cm.gist_ncar
plt.gca().set_color_cycle([colormap(i) for i in np.linspace(0, 0.9, len(params.file_path))])
if params.mask is not None:
params.mask = easy_pickle.load(params.mask)
if imageset is None:
iterable = params.file_path
load_func = lambda x: DataBlockFactory.from_filenames([x])[0].extract_imagesets()[0]
else:
iterable = [imageset]
load_func = lambda x: x
# Iterate over each file provided
for item in iterable:
iset = load_func(item)
n_images = len(iset)
if params.image_number is None:
subiterable = xrange(n_images)
else:
subiterable = [params.image_number]
for image_number in subiterable:
beam = iset.get_beam(image_number)
detector = iset.get_detector(image_number)
s0 = col(beam.get_s0())
# Search the detector for the panel farthest from the beam. The number of bins in the radial average will be
# equal to the farthest point from the beam on the detector, in pixels, unless overridden at the command line
panel_res = [p.get_max_resolution_at_corners(s0) for p in detector]
farthest_panel = detector[panel_res.index(min(panel_res))]
size2, size1 = farthest_panel.get_image_size()
corners = [(0,0), (size1-1,0), (0,size2-1), (size1-1,size2-1)]
corners_lab = [col(farthest_panel.get_pixel_lab_coord(c)) for c in corners]
corner_two_thetas = [farthest_panel.get_two_theta_at_pixel(s0, c) for c in corners]
extent_two_theta = max(corner_two_thetas)
max_corner = corners_lab[corner_two_thetas.index(extent_two_theta)]
extent = int(math.ceil(max_corner.length()*math.sin(extent_two_theta)/max(farthest_panel.get_pixel_size())))
extent_two_theta *= 180/math.pi
if params.n_bins < extent:
params.n_bins = extent
# These arrays will store the radial average info
sums = flex.double(params.n_bins) * 0
sums_sq = flex.double(params.n_bins) * 0
counts = flex.int(params.n_bins) * 0
all_data = iset[image_number]
if not isinstance(all_data, tuple):
all_data = (all_data,)
for tile, (panel, data) in enumerate(zip(detector, all_data)):
if params.mask is None:
mask = flex.bool(flex.grid(data.focus()), True)
else:
mask = params.mask[tile]
if hasattr(data,"as_double"):
data = data.as_double()
logger.flush()
if params.verbose:
logger.write("Average intensity tile %d: %9.3f\n"%(tile, flex.mean(data)))
logger.write("N bins: %d\n"%params.n_bins)
logger.flush()
x1,y1,x2,y2 = 0,0,panel.get_image_size()[1],panel.get_image_size()[0]
bc = panel.get_beam_centre_px(beam.get_s0())
bc = int(round(bc[1])), int(round(bc[0]))
# compute the average
radial_average(data,mask,bc,sums,sums_sq,counts,panel.get_pixel_size()[0],panel.get_distance(),
(x1,y1),(x2,y2))
# average the results, avoiding division by zero
results = sums.set_selected(counts <= 0, 0)
results /= counts.set_selected(counts <= 0, 1).as_double()
if params.median_filter_size is not None:
logger.write("WARNING, the median filter is not fully propogated to the variances\n")
from scipy.ndimage.filters import median_filter
results = flex.double(median_filter(results.as_numpy_array(), size = params.median_filter_size))
# calculate standard devations
stddev_sel = ((sums_sq-sums*results) >= 0) & (counts > 0)
std_devs = flex.double(len(sums), 0)
std_devs.set_selected(stddev_sel,
(sums_sq.select(stddev_sel)-sums.select(stddev_sel)* \
results.select(stddev_sel))/counts.select(stddev_sel).as_double())
std_devs = flex.sqrt(std_devs)
twotheta = flex.double(xrange(len(results)))*extent_two_theta/params.n_bins
q_vals = 4*math.pi*flex.sin(math.pi*twotheta/360)/beam.get_wavelength()
if params.low_max_two_theta_limit is None:
subset = results
else:
subset = results.select(twotheta >= params.low_max_two_theta_limit)
max_result = flex.max(subset)
if params.x_axis == 'two_theta':
xvals = twotheta
max_x = twotheta[flex.first_index(results, max_result)]
elif params.x_axis == 'q':
xvals = q_vals
max_x = q_vals[flex.first_index(results, max_result)]
for i in xrange(len(results)):
val = xvals[i]
if params.output_bins and "%.3f"%results[i] != "nan":
#logger.write("%9.3f %9.3f\n"% (val,results[i])) #.xy format for Rex.cell.
logger.write("%9.3f %9.3f %9.3f\n"%(val,results[i],std_devs[i])) #.xye format for GSASII
#logger.write("%.3f %.3f %.3f\n"%(val,results[i],ds[i])) # include calculated d spacings
logger.write("Maximum %s: %f, value: %f\n"%(params.x_axis, max_x, max_result))
if params.show_plots:
if params.plot_x_max is not None:
results = results.select(xvals <= params.plot_x_max)
xvals = xvals.select(xvals <= params.plot_x_max)
if params.normalize:
plt.plot(xvals.as_numpy_array(),(results/flex.max(results)).as_numpy_array(),'-')
else:
plt.plot(xvals.as_numpy_array(),results.as_numpy_array(),'-')
if params.x_axis == 'two_theta':
plt.xlabel("2 theta")
elif params.x_axis == 'q':
plt.xlabel("q")
plt.ylabel("Avg ADUs")
if params.plot_y_max is not None:
plt.ylim(0, params.plot_y_max)
if params.show_plots:
#plt.legend([os.path.basename(os.path.splitext(f)[0]) for f in params.file_path], ncol=2)
plt.show()
return xvals, results
def get_results(self, finished_objects=None):
if not finished_objects:
finished_objects = self.info.get_finished_objects()
if not finished_objects:
return False
final_objects = []
self.info.unplotted_stats = {}
for key in self.info.stats:
self.info.unplotted_stats[key] = dict(lst=[])
for obj in finished_objects:
item = [obj.input_index, obj.img_path, obj.img_index]
if len(self.info.unprocessed) > 0 and item in self.info.unprocessed:
self.info.unprocessed.remove(item)
if (
len(self.info.categories["not_processed"][0]) > 0
and item in self.info.categories["not_processed"][0]
):
self.info.categories["not_processed"][0].remove(item)
if obj.fail:
key = obj.fail.replace(" ", "_")
if key in self.info.categories:
self.info.categories[key][0].append(item)
else:
self.info.categories["integrated"][0].append(obj.final["final"])
self.info.final_objects.append(obj.obj_file)
final_objects.append(obj)
if not obj.fail or "triage" not in obj.fail:
self.info.categories["have_diffraction"][0].append(obj.img_path)
# Calculate processing stats from final objects
if final_objects:
self.info.pixel_size = final_objects[0].final["pixel_size"]
# Get observations from file
try:
all_obs = ep.load(self.info.idx_file)
except Exception:
all_obs = None
# Collect image processing stats
for obj in final_objects:
for key in self.info.stats:
if key in obj.final:
stat_tuple = (
obj.input_index,
obj.img_path,
obj.img_index,
obj.final[key],
)
self.info.stats[key]["lst"].append(stat_tuple)
# add proc filepath info to 'pointers'
pointer_dict = {
"img_file": obj.img_path,
"obj_file": obj.obj_file,
"img_index": obj.img_index,
"experiments": obj.eint_path,
"reflections": obj.rint_path,
}
self.info.pointers[str(obj.input_index)] = pointer_dict
if key not in self.info.unplotted_stats:
self.info.unplotted_stats[key] = dict(lst=[])
self.info.unplotted_stats[key]["lst"].append(stat_tuple)
# Unit cells and space groups (i.e. cluster iterable)
self.info.cluster_iterable.append(
[
float(obj.final["a"]),
float(obj.final["b"]),
float(obj.final["c"]),
float(obj.final["alpha"]),
float(obj.final["beta"]),
float(obj.final["gamma"]),
str(obj.final["sg"]),
]
)
# Get observations from this image
obs = None
if "observations" in obj.final:
obs = obj.final["observations"].as_non_anomalous_array()
else:
pickle_path = obj.final["final"]
if os.path.isfile(pickle_path):
try:
pickle = ep.load(pickle_path)
obs = pickle["observations"][0].as_non_anomalous_array()
except Exception as e:
print(
"IMAGE_PICKLE_ERROR for {}: {}".format(pickle_path, e)
)
with util.Capturing():
if obs:
# Append observations to combined miller array
obs = obs.expand_to_p1()
if all_obs:
all_obs = all_obs.concatenate(
obs, assert_is_similar_symmetry=False
)
else:
all_obs = obs
# Get B-factor from this image
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(500)
observations_as_f = obs.as_amplitude_array()
observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(
observations_as_f, asu_contents, e_statistics=True
)
b_factor = wp.wilson_b
except RuntimeError as e:
b_factor = 0
print("B_FACTOR_ERROR: ", e)
self.info.b_factors.append(b_factor)
# Save collected observations to file
if all_obs:
ep.dump(self.info.idx_file, all_obs)
# Calculate dataset stats
for k in self.info.stats:
stat_list = list(zip(*self.info.stats[k]["lst"]))[3]
stats = dict(
lst=self.info.stats[k]["lst"],
median=np.median(stat_list).item(),
mean=np.mean(stat_list).item(),
std=np.std(stat_list).item(),
max=np.max(stat_list).item(),
min=np.min(stat_list).item(),
cons=Counter(stat_list).most_common(1)[0][0],
)
self.info.stats[k].update(stats)
return True
else:
return False
def __init__(self, pixel_histograms, output_dirname=".", gain_map_path=None,
gain_map=None, estimated_gain=30,roi=None,run=None):
self.sum_img = flex.double(flex.grid(370,391), 0) # XXX define the image size some other way?
gain_img = flex.double(self.sum_img.accessor(), 0)
assert [gain_map, gain_map_path].count(None) > 0
if gain_map_path is not None:
d = easy_pickle.load(gain_map_path)
gain_map = d["DATA"]
mask = flex.int(self.sum_img.accessor(), 0)
start_row = 370
end_row = 0
print(len(pixel_histograms.histograms))
pixels = list(pixel_histograms.pixels())
n_pixels = len(pixels)
if roi is not None:
for k, (i, j) in enumerate(reversed(pixels)):
if ( i < roi[2]
or i > roi[3]
or j < roi[0]
or j > roi[1]):
del pixels[n_pixels-k-1]
if gain_map is None:
fixed_func = pixel_histograms.fit_one_histogram
else:
def fixed_func(pixel):
return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)
chi_squared_list=flex.double()
for i, pixel in enumerate(pixels):
#print i,pixel
LEG = False
start_row = min(start_row, pixel[0])
end_row = max(end_row, pixel[0])
n_photons = 0
try:
if LEG: gaussians, two_photon_flag = pixel_histograms.fit_one_histogram(pixel)
alt_gaussians = pixel_histograms.fit_one_histogram_two_gaussians(pixel)
except ZeroDivisionError:
print("HEY DIVIDE BY ZERO")
#pixel_histograms.plot_combo(pixel, gaussians)
mask[pixel] = 1
continue
except RuntimeError as e:
print("Error fitting pixel %s" %str(pixel))
print(str(e))
mask[pixel] = 1
continue
hist = pixel_histograms.histograms[pixel]
if not LEG:
gs = alt_gaussians[1].params
fit_photons = gs[0] * gs[2] * math.sqrt(2.*math.pi)
n_photons = int(round(fit_photons,0))
fit_interpretation=pixel_histograms.multiphoton_and_fit_residual(
pixel_histograms.histograms[pixel], alt_gaussians)
multi_photons = fit_interpretation.get_multiphoton_count()
total_photons = n_photons + multi_photons
if False and n_photons< 0: # Generally, do not mask negative values; if fit is still OK
print("\n%d pixel %s altrn %d photons from curvefitting"%( i,pixel,n_photons ))
pixel_histograms.plot_combo(pixel, alt_gaussians,
interpretation=fit_interpretation)
mask[pixel]=1 # do not mask out negative pixels if the Gaussian fit is good
continue
chi_squared_list.append(fit_interpretation.chi_squared())
suspect = False # don't know the optimal statistical test. Histograms vary primarily by total count & # photons
if total_photons <= 3:
if fit_interpretation.chi_squared() > 2.5 or fit_interpretation.quality_factor < 5: suspect=True
elif 3 < total_photons <= 10:
if fit_interpretation.chi_squared() > 5 or fit_interpretation.quality_factor < 10: suspect=True
elif 10 < total_photons <= 33:
if fit_interpretation.chi_squared() > 10 or fit_interpretation.quality_factor < 20: suspect=True
elif 33 < total_photons <= 100:
if fit_interpretation.chi_squared() > 20 or fit_interpretation.quality_factor < 20: suspect=True
elif 100 < total_photons <= 330:
if fit_interpretation.chi_squared() > 30 or fit_interpretation.quality_factor < 25: suspect=True
elif 330 < total_photons <= 1000:
if fit_interpretation.chi_squared() > 40 or fit_interpretation.quality_factor < 30: suspect=True
elif 1000 < total_photons:
if fit_interpretation.chi_squared() > 50 or fit_interpretation.quality_factor < 30: suspect=True
if suspect:
print("\n%d pixel %s Bad quality 0/1-photon fit"%(i,pixel),fit_interpretation.quality_factor)
print(" with chi-squared %10.5f"%fit_interpretation.chi_squared())
print(" Suspect",suspect)
print("%d fit photons, %d total photons"%(n_photons,total_photons))
#pixel_histograms.plot_combo(pixel, alt_gaussians,
# interpretation=fit_interpretation)
mask[pixel]=1
continue
self.sum_img[pixel] = n_photons + multi_photons
mask.set_selected(self.sum_img == 0, 1)
unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
mask.set_selected(unbound_pixel_mask > 0, 1)
bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
mask.set_selected(bad_pixel_mask > 0, 1)
for row in range(self.sum_img.all()[0]):
self.sum_img[row:row+1,:].count(0)
spectrum_focus = self.sum_img[start_row:end_row,:]
mask_focus = mask[start_row:end_row,:]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print("Number of rows: %i" %spectrum_focus.all()[0])
print("Estimated no. photons counted: %i" %flex.sum(spectrum_focus))
print("Number of images used: %i" %flex.sum(
pixel_histograms.histograms.values()[0].slots()))
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr="_%04d"%run
else: runstr=""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_'%runstr)
plot_x, plot_y = xes_finalise.output_spectrum(
spectrum_focus.iround(), mask_focus=mask_focus,
output_dirname=output_dirname, run=run)
self.spectrum = (plot_x, plot_y)
self.spectrum_focus = spectrum_focus
xes_finalise.output_matlab_form(spectrum_focus, "%s/sum%s.m" %(output_dirname,runstr))
print(output_dirname)
print("Average chi squared is",flex.mean(chi_squared_list),"on %d shots"%flex.sum(hist.slots()))
def refine_expanding(params, merged_scope, combine_phil):
assert params.start_at_hierarchy_level == 0
if params.rmsd_filter.enable:
input_name = "filtered"
command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s"
command = command % ("%s_combined_experiments.json" % params.tag,
"%s_combined_reflections.pickle" % params.tag,
"%s_filtered_experiments.json" % params.tag,
"%s_filtered_reflections.pickle" % params.tag)
command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
result.show_stdout()
else:
input_name = "combined"
# this is the order to refine the CSPAD in
steps = {}
steps[0] = [2, 3]
steps[1] = steps[0] + [0, 1]
steps[2] = steps[1] + [14, 15]
steps[3] = steps[2] + [6, 7]
steps[4] = steps[3] + [4, 5]
steps[5] = steps[4] + [12, 13]
steps[6] = steps[5] + [8, 9]
steps[7] = steps[6] + [10, 11]
for s, panels in steps.iteritems():
rest = []
for p in panels:
rest.append(p + 16)
rest.append(p + 32)
rest.append(p + 48)
panels.extend(rest)
levels = {0: (0, 1)} # levels 0 and 1
for i in range(7):
levels[i + 1] = (2, ) # level 2
previous_step_and_level = None
for j in range(8):
from libtbx import easy_pickle
print "Filtering out all reflections except those on panels %s" % (
", ".join(["%d" % p for p in steps[j]]))
combined_path = "%s_%s_reflections.pickle" % (params.tag, input_name)
output_path = "%s_reflections_step%d.pickle" % (params.tag, j)
data = easy_pickle.load(combined_path)
sel = None
for panel_id in steps[j]:
if sel is None:
sel = data['panel'] == panel_id
else:
sel |= data['panel'] == panel_id
print "Retaining", len(
data.select(sel)), "out of", len(data), "reflections"
easy_pickle.dump(output_path, data.select(sel))
for i in levels[j]:
print "Step", j, "refining at hierarchy level", i
refine_phil_file = "%s_refine_step%d_level%d.phil" % (params.tag,
j, i)
if i == 0:
if params.refine_distance:
diff_phil = "refinement.parameterisation.detector.fix_list=Tau1" # fix detector rotz
else:
diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Tau1" # fix detector rotz, distance
if params.flat_refinement:
diff_phil += ",Tau2,Tau3" # Also fix x and y rotations
diff_phil += "\n"
if params.refine_energy:
diff_phil += "refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine
else:
# Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always
if params.flat_refinement and params.flat_refinement_with_distance:
diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation
diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level
elif params.flat_refinement:
diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation
else:
diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything
if previous_step_and_level is None:
command = "dials.refine %s %s_%s_experiments.json %s_reflections_step%d.pickle"%( \
refine_phil_file, params.tag, input_name, params.tag, j)
else:
p_step, p_level = previous_step_and_level
if p_step == j:
command = "dials.refine %s %s_refined_experiments_step%d_level%d.json %s_refined_reflections_step%d_level%d.pickle"%( \
refine_phil_file, params.tag, p_step, p_level, params.tag, p_step, p_level)
else:
command = "dials.refine %s %s_refined_experiments_step%d_level%d.json %s_reflections_step%d.pickle"%( \
refine_phil_file, params.tag, p_step, p_level, params.tag, j)
diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i
output_experiments = "%s_refined_experiments_step%d_level%d.json" % (
params.tag, j, i)
command += " output.experiments=%s output.reflections=%s_refined_reflections_step%d_level%d.pickle"%( \
output_experiments, params.tag, j, i)
scope = merged_scope.fetch(parse(diff_phil))
f = open(refine_phil_file, 'w')
f.write(refine_scope.fetch_diff(scope).as_str())
f.close()
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
result.show_stdout()
# In expanding mode, if using flat refinement with distance, after having refined this step as a block, unrefined
# panels will have been left behind. Read back the new metrology, compute the shift applied to the panels refined
# in this step,and apply that shift to the unrefined panels in this step
if params.flat_refinement and params.flat_refinement_with_distance and i > 0:
from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentListDumper
from xfel.command_line.cspad_detector_congruence import iterate_detector_at_level, iterate_panels
from scitbx.array_family import flex
from scitbx.matrix import col
from libtbx.test_utils import approx_equal
experiments = ExperimentListFactory.from_json_file(
output_experiments, check_format=False)
assert len(experiments.detectors()) == 1
detector = experiments.detectors()[0]
# Displacements: deltas along the vector normal to the detector
displacements = flex.double()
# Iterate through the panel groups at this level
for panel_group in iterate_detector_at_level(
detector.hierarchy(), 0, i):
# Were there panels refined in this step in this panel group?
test = [
list(detector).index(panel) in steps[j]
for panel in iterate_panels(panel_group)
]
if not any(test): continue
# Compute the translation along the normal of this panel group. This is defined as distance in dials.refine
displacements.append(
col(panel_group.get_local_fast_axis()).cross(
col(panel_group.get_local_slow_axis())).dot(
col(panel_group.get_local_origin())))
# Even though the panels are constrained to move the same amount, there is a bit a variation.
stats = flex.mean_and_variance(displacements)
displacement = stats.mean()
print "Average displacement along normals: %f +/- %f" % (
stats.mean(), stats.unweighted_sample_standard_deviation())
# Verify the variation isn't significant
for k in range(1, len(displacements)):
assert approx_equal(displacements[0], displacements[k])
# If all of the panel groups in this level moved, no need to do anything.
if len(displacements) != len(
list(
iterate_detector_at_level(detector.hierarchy(), 0,
i))):
for panel_group in iterate_detector_at_level(
detector.hierarchy(), 0, i):
test = [
list(detector).index(panel) in steps[j]
for panel in iterate_panels(panel_group)
]
# If any of the panels in this panel group moved, no need to do anything
if any(test): continue
# None of the panels in this panel group moved in this step, so need to apply displacement from other panel
# groups at this level
fast = col(panel_group.get_local_fast_axis())
slow = col(panel_group.get_local_slow_axis())
ori = col(panel_group.get_local_origin())
normal = fast.cross(slow)
panel_group.set_local_frame(
fast, slow, (ori.dot(fast) * fast) +
(ori.dot(slow) * slow) + (normal * displacement))
# Check the new displacements. Should be the same across all panels.
displacements = []
for panel_group in iterate_detector_at_level(
detector.hierarchy(), 0, i):
displacements.append(
col(panel_group.get_local_fast_axis()).cross(
col(panel_group.get_local_slow_axis())).dot(
col(panel_group.get_local_origin())))
for k in range(1, len(displacements)):
assert approx_equal(displacements[0], displacements[k])
dump = ExperimentListDumper(experiments)
dump.as_json(output_experiments)
previous_step_and_level = j, i
output_geometry(params)
def test_reindex(dials_regression, run_in_tmpdir):
data_dir = os.path.join(dials_regression, "indexing_test_data",
"i04_weak_data")
pickle_path = os.path.join(data_dir, "indexed.pickle")
experiments_path = os.path.join(data_dir, "experiments.json")
commands = [
"dials.reindex",
pickle_path,
experiments_path,
"change_of_basis_op=2a,b,c",
"space_group=P1",
]
command = " ".join(commands)
print(command)
result = easy_run.fully_buffered(command=command).raise_if_errors()
old_reflections = easy_pickle.load(pickle_path)
assert os.path.exists("reindexed.refl")
new_reflections = easy_pickle.load("reindexed.refl")
old_experiments = load.experiment_list(experiments_path,
check_format=False)
assert os.path.exists("reindexed.expt")
new_experiments = load.experiment_list("reindexed.expt",
check_format=False)
h1, k1, l1 = old_reflections["miller_index"].as_vec3_double().parts()
h2, k2, l2 = new_reflections["miller_index"].as_vec3_double().parts()
assert 2 * h1 == pytest.approx(h2)
assert k1 == pytest.approx(k2)
assert l1 == pytest.approx(l2)
old_uc_params = old_experiments[0].crystal.get_unit_cell().parameters()
new_uc_params = new_experiments[0].crystal.get_unit_cell().parameters()
assert new_uc_params[0] == pytest.approx(2 * old_uc_params[0])
assert new_uc_params[1:] == pytest.approx(old_uc_params[1:])
assert old_experiments[0].crystal.get_space_group().type().hall_symbol(
) == " P 1"
assert new_experiments[0].crystal.get_space_group().type().hall_symbol(
) == " P 1"
# set space group P4
cb_op = sgtbx.change_of_basis_op("a,b,c")
commands = [
"dials.reindex",
experiments_path,
"space_group=P4",
"change_of_basis_op=%s" % str(cb_op),
"output.experiments=P4.expt",
]
command = " ".join(commands)
print(command)
result = easy_run.fully_buffered(command=command).raise_if_errors()
# apply one of the symops from the space group
cb_op = sgtbx.change_of_basis_op("-x,-y,z")
commands = [
"dials.reindex",
"P4.expt",
"change_of_basis_op=%s" % str(cb_op),
"output.experiments=P4_reindexed.expt",
]
command = " ".join(commands)
print(command)
result = easy_run.fully_buffered(command=command).raise_if_errors()
new_experiments1 = load.experiment_list("P4_reindexed.expt",
check_format=False)
assert new_experiments1[0].crystal.get_A() == pytest.approx(
old_experiments[0].crystal.change_basis(cb_op).get_A())
#
cb_op = sgtbx.change_of_basis_op("-x,-y,z")
commands = [
"dials.reindex",
"P4.expt",
"change_of_basis_op=auto",
"reference.experiments=P4_reindexed.expt",
"output.experiments=P4_reindexed2.expt",
]
command = " ".join(commands)
print(command)
result = easy_run.fully_buffered(command=command).raise_if_errors()
new_experiments2 = load.experiment_list("P4_reindexed2.expt",
check_format=False)
assert new_experiments1[0].crystal.get_A() == pytest.approx(
new_experiments2[0].crystal.get_A())
def test_reindex_against_reference(dials_regression, tmpdir):
"""Test the reindexing against a reference dataset functionality."""
tmpdir.chdir()
data_dir = os.path.join(dials_regression, "indexing_test_data",
"i04_weak_data")
pickle_path = os.path.join(data_dir, "indexed.pickle")
experiments_path = os.path.join(data_dir, "experiments.json")
commands = [
"dials.reindex",
pickle_path,
experiments_path,
"change_of_basis_op=a,b,c",
"space_group=P4",
"output.reflections=P4.refl",
"output.experiments=P4.expt",
]
command = " ".join(commands)
print(command)
_ = easy_run.fully_buffered(command=command).raise_if_errors()
assert os.path.exists("P4.refl")
assert os.path.exists("P4.expt")
new_experiments = load.experiment_list("P4.expt", check_format=False)
assert new_experiments[0].crystal.get_space_group().type().hall_symbol(
) == " P 4"
# Now have something in P4, get another dataset in a different indexing scheme
cb_op = sgtbx.change_of_basis_op("a,-b,-c")
commands = [
"dials.reindex",
"P4.refl",
"P4.expt",
"change_of_basis_op=%s" % str(cb_op),
"output.experiments=P4_reindexed.expt",
"output.reflections=P4_reindexed.refl",
]
command = " ".join(commands)
print(command)
_ = easy_run.fully_buffered(command=command).raise_if_errors()
# now run reference reindexing
commands = [
"dials.reindex",
"P4.refl",
"P4.expt",
"reference.experiments=P4_reindexed.expt",
"reference.reflections=P4_reindexed.refl",
]
command = " ".join(commands)
print(command)
_ = easy_run.fully_buffered(command=command).raise_if_errors()
# expect reindexed_reflections to be same as P4_reindexed, not P4_reflections
reindexed_reflections = easy_pickle.load("reindexed.refl")
P4_reindexed = easy_pickle.load("P4_reindexed.refl")
P4_reflections = easy_pickle.load("P4.refl")
h1, k1, l1 = reindexed_reflections["miller_index"].as_vec3_double().parts()
h2, k2, l2 = P4_reindexed["miller_index"].as_vec3_double().parts()
h3, k3, l3 = P4_reflections["miller_index"].as_vec3_double().parts()
# hkl1 and hkl2 should be same, as should have been reindexed by against the
# reference, with the program determing a reindexing operator of a,-b,-c
assert list(h1) == pytest.approx(list(h2))
assert list(l1) == pytest.approx(list(l2))
assert list(k1) == pytest.approx(list(k2))
# h1 and h3 should be same, but not l and k, as these dataset should differ
# by an a twinning operator of a,-b,-c
assert list(h1) == pytest.approx(list(h3))
assert list(l1) != pytest.approx(list(l3))
assert list(k1) != pytest.approx(list(k3))
def stats(model, prefix, no_ticks=True):
# Get rid of H, multi-model, no-protein and single-atom residue models
if (model.percent_of_single_atom_residues() > 20):
return None
sel = model.selection(string="protein")
if (sel.count(True) == 0):
return None
ssr = "protein and not (element H or element D or resname UNX or resname UNK or resname UNL)"
sel = model.selection(string=ssr)
model = model.select(sel)
if (len(model.get_hierarchy().models()) > 1):
return None
# Add H; this looses CRYST1 !
rr = run_reduce_with_timeout(
stdin_lines=model.get_hierarchy().as_pdb_string().splitlines(),
file_name=None,
parameters="-oh -his -flip -keep -allalt -pen9999 -",
override_auto_timeout_with=None)
# Create model; this is a single-model pure protein with new H added
pdb_inp = iotbx.pdb.input(source_info=None, lines=rr.stdout_lines)
model = mmtbx.model.manager(model_input=None,
build_grm=True,
pdb_hierarchy=pdb_inp.construct_hierarchy(),
process_input=True,
log=null_out())
box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center(
sites_cart=model.get_sites_cart(), buffer_layer=5)
model.set_sites_cart(box.sites_cart)
model._crystal_symmetry = box.crystal_symmetry()
#
N = 10
SS = get_ss_selections(hierarchy=model.get_hierarchy())
HB_all = find(
model=model.select(flex.bool(model.size(), True)),
a_DHA_cutoff=90).get_params_as_arrays(replace_with_empty_threshold=N)
HB_alpha = find(
model=model.select(SS.both.h_sel),
a_DHA_cutoff=90).get_params_as_arrays(replace_with_empty_threshold=N)
HB_beta = find(
model=model.select(SS.both.s_sel),
a_DHA_cutoff=90).get_params_as_arrays(replace_with_empty_threshold=N)
print(HB_all.d_HA.size())
result_dict = {}
result_dict["all"] = HB_all
result_dict["alpha"] = HB_alpha
result_dict["beta"] = HB_beta
# result_dict["loop"] = get_selected(sel=loop_sel)
# Load histograms for reference high-resolution d_HA and a_DHA
pkl_fn = libtbx.env.find_in_repositories(
relative_path="mmtbx") + "/nci/d_HA_and_a_DHA_high_res.pkl"
assert os.path.isfile(pkl_fn)
ref = easy_pickle.load(pkl_fn)
#
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(10, 10))
kwargs = dict(histtype='bar', bins=20, range=[1.6, 3.0], alpha=.8)
for j, it in enumerate([["alpha", 1], ["beta", 3], ["all", 5]]):
key, i = it
ax = plt.subplot(int("32%d" % i))
if (no_ticks):
#ax.set_xticks([])
ax.set_yticks([])
if (j in [0, 1]):
ax.tick_params(bottom=False)
ax.set_xticklabels([])
ax.tick_params(axis="x", labelsize=12)
ax.tick_params(axis="y", labelsize=12, left=False, pad=-2)
ax.text(0.98,
0.92,
key,
size=12,
horizontalalignment='right',
transform=ax.transAxes)
HB = result_dict[key]
if HB is None: continue
w1 = np.ones_like(HB.d_HA) / HB.d_HA.size()
ax.hist(HB.d_HA, color="orangered", weights=w1, rwidth=0.3, **kwargs)
#
start, end1, end2 = 0, max(ref.distances[key].vals), \
round(max(ref.distances[key].vals),2)
if (not no_ticks):
plt.yticks([0.01, end1], ["0", end2],
visible=True,
rotation="horizontal")
if (key == "alpha"): plt.ylim(0, end2 + 0.02)
elif (key == "beta"): plt.ylim(0, end2 + 0.02)
elif (key == "all"): plt.ylim(0, end2 + 0.02)
else: assert 0
#
if (j == 0): ax.set_title("Distance", size=15)
bins = list(flex.double(ref.distances[key].bins))
ax.bar(bins, ref.distances[key].vals, alpha=.3, width=0.07)
#
kwargs = dict(histtype='bar', bins=20, range=[90, 180], alpha=.8)
for j, it in enumerate([["alpha", 2], ["beta", 4], ["all", 6]]):
key, i = it
ax = plt.subplot(int("32%d" % i))
if (j in [0, 1]):
ax.tick_params(bottom=False)
ax.set_xticklabels([])
if (no_ticks):
#ax.set_xticks([])
ax.set_yticks([])
ax.tick_params(axis="x", labelsize=12)
ax.tick_params(axis="y", labelsize=12, left=False, pad=-2)
ax.text(0.98,
0.92,
key,
size=12,
horizontalalignment='right',
transform=ax.transAxes)
ax.text(0.98,
0.92,
key,
size=12,
horizontalalignment='right',
transform=ax.transAxes)
#if(j in [0,1]): ax.plot_params(bottom=False)
HB = result_dict[key]
if HB is None: continue
w1 = np.ones_like(HB.a_DHA) / HB.a_DHA.size()
ax.hist(HB.a_DHA, color="orangered", weights=w1, rwidth=0.3, **kwargs)
#
start, end1, end2 = 0, max(ref.angles[key].vals), \
round(max(ref.angles[key].vals),2)
if (not no_ticks):
plt.yticks([0.01, end1], ["0", end2],
visible=True,
rotation="horizontal")
if (key == "alpha"): plt.ylim(0, end2 + 0.02)
elif (key == "beta"): plt.ylim(0, end2 + 0.02)
elif (key == "all"): plt.ylim(0, end2 + 0.02)
else: assert 0
#
if (j == 0): ax.set_title("Angle", size=15)
ax.bar(ref.angles[key].bins, ref.angles[key].vals, width=4.5, alpha=.3)
plt.subplots_adjust(wspace=0.12, hspace=0.025)
if (no_ticks):
plt.subplots_adjust(wspace=0.025, hspace=0.025)
#fig.savefig("%s.png"%prefix, dpi=1000)
fig.savefig("%s.pdf" % prefix)
def __init__(self,
address,
avg_dirname=None,
avg_basename=None,
stddev_dirname=None,
stddev_basename=None,
max_dirname=None,
max_basename=None,
background_path=None,
flags=None,
hot_threshold=None,
gain_threshold=None,
noise_threshold=7,
elastic_threshold=9,
symnoise_threshold=4,
**kwds):
"""
@param address Full data source address of the DAQ device
@param avg_dirname Directory portion of output average image
XXX mean
@param avg_basename Filename prefix of output average image XXX
mean
@param flags inactive: Eliminate the inactive pixels
noelastic: Eliminate elastic scattering
nohot: Eliminate the hot pixels
nonoise: Eliminate noisy pixels
symnoise: Symmetrically eliminate noisy pixels
@param stddev_dirname Directory portion of output standard
deviation image XXX std
@param stddev_basename Filename prefix of output standard
deviation image XXX std
@param max_dirname Directory portion of output maximum
projection image
@param max_basename Filename prefix of output maximum
projection image
"""
super(average_mixin, self).__init__(
address=address,
**kwds
)
self.roi = None
self.avg_basename = cspad_tbx.getOptString(avg_basename)
self.avg_dirname = cspad_tbx.getOptString(avg_dirname)
self.detector = cspad_tbx.address_split(address)[0]
self.flags = cspad_tbx.getOptStrings(flags, default = [])
self.stddev_basename = cspad_tbx.getOptString(stddev_basename)
self.stddev_dirname = cspad_tbx.getOptString(stddev_dirname)
self.max_basename = cspad_tbx.getOptString(max_basename)
self.max_dirname = cspad_tbx.getOptString(max_dirname)
self.background_path = cspad_tbx.getOptString(background_path)
self.hot_threshold = cspad_tbx.getOptFloat(hot_threshold)
self.gain_threshold = cspad_tbx.getOptFloat(gain_threshold)
self.noise_threshold = cspad_tbx.getOptFloat(noise_threshold)
self.elastic_threshold = cspad_tbx.getOptFloat(elastic_threshold)
self.symnoise_threshold = cspad_tbx.getOptFloat(symnoise_threshold)
if background_path is not None:
background_dict = easy_pickle.load(background_path)
self.background_img = background_dict['DATA']
self._have_max = self.max_basename is not None or \
self.max_dirname is not None
self._have_mean = self.avg_basename is not None or \
self.avg_dirname is not None
self._have_std = self.stddev_basename is not None or \
self.stddev_dirname is not None
# Start a server process which holds a set of Python objects that
# other processes can manipulate using proxies. The queues will
# be used in endjob() to pass images between the worker processes,
# and the lock will ensure the transfer is treated as a critical
# section. There is therefore the risk of a hang if the queues
# cannot hold all the data one process will supply before another
# empties it.
#
# In an attempt to alleviate this issue, separate queues are used
# for the potentially big images. The hope is to prevent
# producers from blocking while consumers are locked out by using
# more buffers.
mgr = multiprocessing.Manager()
self._lock = mgr.Lock()
self._metadata = mgr.dict()
self._queue_max = mgr.Queue()
self._queue_sum = mgr.Queue()
self._queue_ssq = mgr.Queue()
def read_object_file(self, filepath):
try:
object = ep.load(filepath)
return object
except EOFError:
pass
def run (args, source_data = None) :
from xfel import radial_average
from scitbx.array_family import flex
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function
from spotfinder.applications.xfel import cxi_phil
from iotbx.detectors.npy import NpyImage
import os, sys
from iotbx.detectors.npy import NpyImage
user_phil = []
# TODO: replace this stuff with iotbx.phil.process_command_line_with_files
# as soon as I can safely modify it
for arg in args :
if (not "=" in arg) :
try :
user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg))
except ValueError as e :
raise Sorry("Unrecognized argument '%s'" % arg)
else :
try :
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError as e :
raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if params.file_path is None or not os.path.isfile(params.file_path) and source_data is None:
master_phil.show()
raise Usage("file_path must be defined (either file_path=XXX, or the path alone).")
assert params.handedness is not None
assert params.n_bins is not None
assert params.verbose is not None
assert params.output_bins is not None
if source_data is None:
from libtbx import easy_pickle
source_data = easy_pickle.load(params.file_path)
if params.output_file is None:
logger = sys.stdout
else:
logger = open(params.output_file, 'w')
logger.write("%s "%params.output_file)
if not "DETECTOR_ADDRESS" in source_data:
# legacy format; try to guess the address
LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
LCLS_detector_address = source_data["DETECTOR_ADDRESS"]
timesec = reverse_timestamp( source_data["TIMESTAMP"] )[0]
version_lookup = detector_format_function(LCLS_detector_address,timesec)
args = [
"distl.detector_format_version=%s"%version_lookup,
"viewer.powder_arcs.show=False",
"viewer.powder_arcs.code=3n9c",
]
horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands
img = NpyImage(params.file_path, source_data)
img.readHeader(horizons_phil)
img.translate_tiles(horizons_phil)
if params.verbose:
img.show_header()
the_tiles = img.get_tile_manager(horizons_phil).effective_tiling_as_flex_int(
reapply_peripheral_margin=False,encode_inactive_as_zeroes=True)
if params.beam_x is None:
params.beam_x = img.beamx / img.pixel_size
if params.beam_y is None:
params.beam_y = img.beamy / img.pixel_size
if params.verbose:
logger.write("I think the beam center is (%s,%s)\n"%(params.beam_x, params.beam_y))
bc = (int(params.beam_x),int(params.beam_y))
extent = int(math.ceil(max(distance((0,0),bc),
distance((img.size1,0),bc),
distance((0,img.size2),bc),
distance((img.size1,img.size2),bc))))
if params.n_bins < extent:
params.n_bins = extent
extent_in_mm = extent * img.pixel_size
extent_two_theta = math.atan(extent_in_mm/img.distance)*180/math.pi
sums = flex.double(params.n_bins) * 0
sums_sq = flex.double(params.n_bins) * 0
counts = flex.int(params.n_bins) * 0
data = img.get_raw_data()
if hasattr(data,"as_double"):
data = data.as_double()
logger.write("Average intensity: %9.3f\n"%flex.mean(data))
if params.verbose:
logger.write("Generating average...tile:")
logger.flush()
for tile in range(len(the_tiles)//4):
if params.verbose:
logger.write(" %d"%tile)
logger.flush()
x1,y1,x2,y2 = get_tile_coords(the_tiles,tile)
radial_average(data,bc,sums,sums_sq,counts,img.pixel_size,img.distance,
(x1,y1),(x2,y2))
if params.verbose:
logger.write(" Finishing...\n")
# average, avoiding division by zero
results = sums.set_selected(counts <= 0, 0)
results /= counts.set_selected(counts <= 0, 1).as_double()
# calculte standard devations
std_devs = [math.sqrt((sums_sq[i]-sums[i]*results[i])/counts[i])
if counts[i] > 0 else 0 for i in range(len(sums))]
xvals = flex.double(len(results))
max_twotheta = float('-inf')
max_result = float('-inf')
for i in range(len(results)):
twotheta = i * extent_two_theta/params.n_bins
xvals[i] = twotheta
if params.output_bins and "%.3f"%results[i] != "nan":
#logger.write("%9.3f %9.3f\n"% (twotheta,results[i])) #.xy format for Rex.cell.
logger.write("%9.3f %9.3f %9.3f\n"%(twotheta,results[i],std_devs[i])) #.xye format for GSASII
#logger.write("%.3f %.3f %.3f\n"%(twotheta,results[i],ds[i])) # include calculated d spacings
if results[i] > max_result:
max_twotheta = twotheta
max_result = results[i]
logger.write("Maximum 2theta for %s, TS %s: %f, value: %f\n"%(params.file_path, source_data['TIMESTAMP'], max_twotheta, max_result))
if params.verbose:
from pylab import scatter, show, xlabel, ylabel, ylim
scatter(xvals,results)
xlabel("2 theta")
ylabel("Avg ADUs")
if params.plot_y_max is not None:
ylim(0, params.plot_y_max)
show()
return xvals, results
def onRecovery(self, e):
# Find finished runs and display results
int_folder = os.path.abspath('{}/integration'.format(os.curdir))
if not os.path.isdir(int_folder):
open_dlg = wx.DirDialog(self,
"Choose the integration folder:",
style=wx.DD_DEFAULT_STYLE)
if open_dlg.ShowModal() == wx.ID_OK:
int_folder = open_dlg.GetPath()
open_dlg.Destroy()
else:
open_dlg.Destroy()
return
paths = [os.path.join(int_folder, p) for p in os.listdir(int_folder)]
paths = [p for p in paths if os.path.isdir(p)]
path_dlg = dlg.RecoveryDialog(self)
path_dlg.insert_paths(paths)
if path_dlg.ShowModal() == wx.ID_OK:
self.reset_settings()
selected = path_dlg.selected
recovery_mode = path_dlg.recovery_mode
int_path = selected[1]
init_file = os.path.join(int_path, 'init.cfg')
if os.path.isfile(init_file):
rec_init = ep.load(init_file)
tmp_phil = inp.master_phil.format(
python_object=rec_init.params)
self.iota_phil = self.iota_phil.fetch(source=tmp_phil)
else:
rec_init = InitAll(iver=iota_version)
rec_init.int_base = int_path
rec_init.obj_base = os.path.join(int_path, 'image_objects')
rec_init.fin_base = os.path.join(int_path, 'final')
rec_init.log_base = os.path.join(int_path, 'logs')
rec_init.viz_base = os.path.join(int_path, 'visualization')
rec_init.logfile = os.path.join(int_path, 'iota.log')
with open(rec_init.logfile, 'r') as lf:
lines = lf.readlines()[4:86]
log_phil = ip.parse(''.join(lines))
self.iota_phil = self.iota_phil.fetch(source=log_phil)
rec_init.params = self.iota_phil.extract()
input_entries = [i for i in rec_init.params.input if i != None]
rec_init.input_list = ginp.make_input_list(input_entries)
self.gparams = self.iota_phil.extract()
# Re-populate input window with settings from read-in run (check that
# nothing has been moved)
rec_target_phil_file = os.path.join(rec_init.int_base,
'target.phil')
with open(rec_target_phil_file, 'r') as pf:
rec_target_phil = pf.read()
self.target_phil = rec_target_phil
self.update_input_window()
# Re-open processing window with results of the run
if recovery_mode == 0:
self.proc_window = frm.ProcWindow(self,
-1,
title='Image Processing',
target_phil=rec_target_phil,
phil=self.iota_phil)
self.proc_window.recover(int_path=rec_init.int_base,
init=rec_init,
status=selected[0],
params=self.gparams)
self.proc_window.Show(True)
def make_png(image_pickle,
integration_pickle,
file_name=None,
res=600,
show_spots=True):
""" Write a png file visualizing integration results.
:param image_pickle: path to image pickle file.
:param integration_pickle: path to integration pickle file.
:param res: resolution of output file in dpi (6x6 image size).
:param show_spots: Uses the `obsspot` field from the `correction_vector` key to plot spot positions. Fails silently if `correction_vector` does not exist. Use option indexing.verbose_cv=True in cxi.index to make sure that this is created.
:file_name: desired output file name. Defaults to the integration_pickle name.
"""
if file_name is None:
import os
# Change extension of `image_pickle` to .png
file_name = os.path.splitext(integration_pickle)[0] + ".png"
# Load image pickle, and convert to image
img_dict = ep.load(image_pickle)
img_data = img_dict['DATA'].as_numpy_array()
# Load integration pickle, and get coordinates of predictions
int_d = ep.load(integration_pickle)
predictions = int_d["mapped_predictions"][0]
pred_coords = predictions.as_double().as_numpy_array() \
.reshape(2, len(predictions), order='F')
if show_spots and 'correction_vectors' in int_d:
spot_coords = [x['obsspot'] for x in int_d['correction_vectors'][0]]
spot_coords = np.array(spot_coords).transpose()
plot_spots = True
else:
plot_spots = False
# Get some other useful info from integration pickle
point_group = int_d['pointgroup']
unit_cell = int_d['current_orientation'][0].unit_cell().parameters()
unit_cell = ', '.join("{:.1f}".format(u) for u in unit_cell)
mosaicity = int_d['mosaicity']
# Create the figure
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(6, 6))
# 1st set of axes for the image
ax = fig.add_subplot(1, 1, 1)
ax.set_xlim(0, len(img_data[1]))
ax.set_ylim(0, len(img_data[0]))
ax.set_aspect('equal')
# To display image properly, need to set img limit to 0.01th percentile
# a few maxed-out pixels will make the entire image appear blank white
clim = (img_data.min(), np.percentile(img_data, 99.99))
ax.imshow(img_data, origin=None, cmap='Greys', clim=clim)
# 2nd set of axes for the predictions
ax2 = fig.add_axes(ax.get_position(), frameon=False) # superimposed axes
ax2.set_xlim(0, len(img_data[1]))
ax2.set_ylim(0, len(img_data[0]))
ax2.set_aspect('equal')
ax2.plot(pred_coords[1],
pred_coords[0],
ms=2,
linestyle='.',
marker='o',
alpha=0.3,
markeredgecolor='r',
markerfacecolor='none',
markeredgewidth=0.3)
if plot_spots:
ax2.plot(spot_coords[1],
spot_coords[0],
ms=2,
linestyle='.',
marker='d',
alpha=0.3,
markeredgecolor='b',
markerfacecolor='none',
markeredgewidth=0.3,
linewidth=1)
plt.title("Unit cell: {} ({}) \nNominal mosaicity: {}" \
.format(point_group, unit_cell, mosaicity))
plt.savefig(file_name, dpi=res, format='png')
def refine_hierarchical(params, merged_scope, combine_phil):
if params.panel_filter is not None:
from libtbx import easy_pickle
print "Filtering out all reflections except those on panels %s" % (
", ".join(["%d" % p for p in params.panel_filter]))
combined_path = "%s_combined_reflections.pickle" % params.tag
data = easy_pickle.load(combined_path)
sel = None
for panel_id in params.panel_filter:
if sel is None:
sel = data['panel'] == panel_id
else:
sel |= data['panel'] == panel_id
print "Retaining", len(
data.select(sel)), "out of", len(data), "reflections"
easy_pickle.dump(combined_path, data.select(sel))
for i in xrange(params.start_at_hierarchy_level,
params.refine_to_hierarchy_level + 1):
if params.rmsd_filter.enable:
input_name = "filtered"
else:
if i == params.start_at_hierarchy_level:
input_name = "combined"
else:
input_name = "refined"
if params.rmsd_filter.enable:
command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s"
if i == params.start_at_hierarchy_level:
command = command % (
"%s_combined_experiments.json" % params.tag,
"%s_combined_reflections.pickle" % params.tag,
"%s_filtered_experiments.json" % params.tag,
"%s_filtered_reflections.pickle" % params.tag)
else:
command = command % ("%s_refined_experiments_level%d.json" %
(params.tag, i - 1),
"%s_refined_reflections_level%d.pickle" %
(params.tag, i - 1),
"%s_filtered_experiments_level%d.json" %
(params.tag, i - 1),
"%s_filtered_reflections_level%d.pickle" %
(params.tag, i - 1))
command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
result.show_stdout()
print "Refining at hierarchy level", i
refine_phil_file = "%s_refine_level%d.phil" % (params.tag, i)
if i == 0:
fix_list = ['Tau1'] # fix detector rotz
if not params.refine_distance:
fix_list.append('Dist')
if params.flat_refinement:
fix_list.extend(['Tau2', 'Tau3'])
diff_phil = "refinement.parameterisation.detector.fix_list=%s\n" % ",".join(
fix_list)
if params.refine_energy:
diff_phil += " refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine
else:
# Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always
if params.flat_refinement and params.flat_refinement_with_distance:
diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation
diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level
elif params.flat_refinement:
diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation
else:
diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything
if i == params.start_at_hierarchy_level:
command = "dials.refine %s %s_%s_experiments.json %s_%s_reflections.pickle" % (
refine_phil_file, params.tag, input_name, params.tag,
input_name)
else:
command = "dials.refine %s %s_%s_experiments_level%d.json %s_%s_reflections_level%d.pickle" % (
refine_phil_file, params.tag, input_name, i - 1, params.tag,
input_name, i - 1)
diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i
command += " output.experiments=%s_refined_experiments_level%d.json output.reflections=%s_refined_reflections_level%d.pickle"%( \
params.tag, i, params.tag, i)
scope = merged_scope.fetch(parse(diff_phil))
f = open(refine_phil_file, 'w')
f.write(refine_scope.fetch_diff(scope).as_str())
f.close()
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
result.show_stdout()
output_geometry(params)
def load_data():
codes = easy_pickle.load(prefix + 'pisa.codes')
codes = flex.std_string(codes)
moments = easy_pickle.load(prefix + 'pisa.nlm')
return codes, moments
def __init__(self, models, log):
db_path = libtbx.env.find_in_repositories(
relative_path="chem_data/rama_z/top8000_rama_z_dict.pkl",
test=os.path.isfile)
self.log = log
# this takes ~0.15 seconds, so I don't see a need to cache it somehow.
self.db = easy_pickle.load(db_path)
# =========================================================================
# change keys in pickle to Python 3 string
# very temporary fix until pickle is updated
if sys.version_info.major == 3:
from libtbx.utils import to_str
for key in list(self.db.keys()):
self.db[to_str(key)] = self.db[key]
for subkey in list(self.db[key].keys()):
self.db[to_str(key)][to_str(subkey)] = self.db[key][subkey]
# =========================================================================
self.calibration_values = {
'H': (-0.045355950779513175, 0.1951165524439217),
'S': (-0.0425581278436754, 0.20068584887814633),
'L': (-0.018457764754231075, 0.15788374669456848),
'W': (-0.016806654295023003, 0.12044960331869274)
}
self.residue_counts = {"H": 0, "S": 0, "L": 0}
self.z_score = {"H": None, "S": None, "L": None, 'W': None}
self.means = {"H": {}, "S": {}, "L": {}}
self.stds = {"H": {}, "S": {}, "L": {}}
self.phi_step = 4
self.psi_step = 4
self.n_phi_half = 45
self.n_psi_half = 45
# this is needed to disable e.g. selection functionality when
# multiple models are present
self.n_models = len(models)
self.res_info = []
for model in models:
if model.get_hierarchy().models_size() > 1:
hierarchy = iotbx.pdb.hierarchy.root()
m = model.get_hierarchy().models()[0].detached_copy()
hierarchy.append_model(m)
asc = hierarchy.atom_selection_cache()
else:
hierarchy = model.get_hierarchy()
asc = model.get_atom_selection_cache()
sec_str_master_phil = iotbx.phil.parse(sec_str_master_phil_str)
ss_params = sec_str_master_phil.fetch().extract()
ss_params.secondary_structure.protein.search_method = "from_ca"
ss_params.secondary_structure.from_ca_conservative = True
ssm = ss_manager(
hierarchy,
atom_selection_cache=asc,
geometry_restraints_manager=None,
sec_str_from_pdb_file=None,
# params=None,
params=ss_params.secondary_structure,
was_initialized=False,
mon_lib_srv=None,
verbose=-1,
log=null_out(),
# log=sys.stdout,
)
filtered_ann = ssm.actual_sec_str.deep_copy()
filtered_ann.remove_short_annotations(
helix_min_len=4,
sheet_min_len=4,
keep_one_stranded_sheets=True)
self.helix_sel = asc.selection(
filtered_ann.overall_helices_selection())
self.sheet_sel = asc.selection(
filtered_ann.overall_sheets_selection())
used_atoms = set()
for three in generate_protein_threes(hierarchy=hierarchy,
geometry=None):
main_residue = three[1]
phi_psi_atoms = three.get_phi_psi_atoms()
if phi_psi_atoms is None:
continue
phi_atoms, psi_atoms = phi_psi_atoms
key = [x.i_seq for x in phi_atoms] + [psi_atoms[-1].i_seq]
key = "%s" % key
if key not in used_atoms:
phi, psi = three.get_phi_psi_angles()
rkey = three.get_ramalyze_key()
resname = main_residue.resname
ss_type = self._figure_out_ss(three)
self.res_info.append(
["", rkey, resname, ss_type, phi, psi])
self.residue_counts[ss_type] += 1
used_atoms.add(key)
self.residue_counts["W"] = self.residue_counts[
"H"] + self.residue_counts["S"] + self.residue_counts["L"]
def find_relatives(ids, cc_min, cc_max, rmax, codes, moments, nmax=10):
indices = flex.int()
idlist = open('id_list.txt', 'r')
for id in idlist:
id = id[0:4]
indices.append(flex.first_index(codes, id))
r_max = easy_pickle.load(prefix + 'pisa.rmax')
nns = easy_pickle.load(prefix + 'pisa.nn')
nn_array = math.nl_array(nmax)
nn_indx = nn_array.nl()
nn_total = nn_indx.size()
q_array = flex.double(range(501)) / 2000.0
ref_nlm_array = math.nlm_array(nmax)
target_nlm_array = math.nlm_array(nmax)
nlm = ref_nlm_array.nlm()
coef_size = nlm.size()
all_indices = range(codes.size())
small_q_array = flex.double(range(51)) / 300.0
mean = []
sig = []
for indx in indices:
print indx
#rmax = 50.0 #r_max[indx]
ref_coef = moments[indx]
ref_nlm_array.load_coefs(nlm, ref_coef[0:coef_size])
z_model = zernike_model(ref_nlm_array, q_array, rmax, nmax)
out_name = codes[indx] + "_.qi"
nn_array.load_coefs(nn_indx, nns[indx][0:nn_total])
ref_int = put_intensity(z_model, q_array, nn_array, out_name)
mean_r = ref_int * 0.0
sig_r = ref_int * 0.0
small_z_model = zernike_model(ref_nlm_array, small_q_array, rmax, nmax)
small_ref_int = small_z_model.calc_intensity(nn_array)
small_ref_int = small_ref_int / small_ref_int[0]
N = 0.0
for coef, ii in zip(moments, all_indices):
if N > 25: break
target_nlm_array.load_coefs(nlm, coef[0:coef_size])
align_obj = fft_align.align(ref_nlm_array,
target_nlm_array,
nmax=nmax,
topn=10,
refine=False)
cc = align_obj.get_cc()
if (cc >= cc_min and cc <= cc_max):
N += 1
nn_array.load_coefs(nn_indx, nns[ii][0:nn_total])
opt_r_obj = optimize_r(nn_array, small_ref_int, small_q_array,
nmax)
opt_r = gss(opt_r_obj.target, rmax * 0.8, rmax * 1.2)
z_model = zernike_model(ref_nlm_array, q_array, opt_r, nmax)
out_name = codes[indx] + "_" + codes[ii] + ".qi.rel"
mod_int = put_intensity(z_model, q_array, nn_array, out_name,
ref_int)
out_name = codes[indx] + "_" + codes[ii] + ".qi"
put_intensity(z_model, q_array, nn_array, out_name)
mod_int = mod_int - 1.0
mean_r += mod_int
sig_r += mod_int * mod_int
print ii, cc, codes[ii], opt_r
if N > 3:
mean_r /= N
sig_r = sig_r / N - mean_r * mean_r
mean.append(mean_r)
sig.append(sig_r)
N = len(mean)
if N > 0:
mean_r = mean[0] * 0.0
s_r = mean[0] * 0.0
for uu in range(N):
mean_r += mean[uu]
s_r += sig[uu]
mean_r /= N
s_r /= N
s_r = flex.sqrt(s_r)
f = open('q_m_s_%s.dat' % rmax, 'w')
for q, m, s in zip(q_array, mean_r, s_r):
print >> f, q, m, s
metro_style = "cbf"
from xfel.cftbx.detector.cspad_cbf_tbx import cbf_file_to_basis_dict
metro = cbf_file_to_basis_dict(params.new_metrology)
else:
metro_style = "flatfile"
from xfel.cftbx.detector.cspad_cbf_tbx import read_optical_metrology_from_flat_file, asic_dimension, asic_gap
metro = read_optical_metrology_from_flat_file(params.new_metrology, params.detector, img['PIXEL_SIZE'],
asic_dimension, asic_gap, plot=params.plot)
for filename in params.pickle_file:
# Read the pickle file and pull the tiles out of it
img = easy_pickle.load(filename)
tiles = {}
asics = {}
data = img['DATA']
if os.path.isdir(params.old_metrology):
num_sections = len(sections)
for p in range(num_sections):
for s in range(len(sections[p])):
# Pull the sensor block from the image, and rotate it back to
# the "lying down" convention.
c = sections[p][s].corners_asic()
k = (int(round(-sections[p][s].angle / 90.0)) + 1) % 4
def get_folders(self):
pdb_code_list = list()
skip_data = dict()
if self.params.models is not None:
for m in self.params.models:
pdb_code_list.append(m.pdb_code)
size = flex.double()
folders = list()
for root, dirs, files in os.walk(cryodir):
if os.path.basename(root) == 'maps_and_models': continue
if os.path.basename(root) == 'unique': continue
# DEBUG
#if (len(folders)>10): break
#
prefix = os.path.basename(root)
#
pdb_code = prefix[0:4]
if pdb_code_list:
if pdb_code not in pdb_code_list: continue
# Check if pickle file exists
pkl_file = os.path.join(root, prefix + '.pkl')
if (not os.path.isfile(pkl_file)):
print('No pickle file for ', prefix, file=self.logger)
skip_data[prefix] = 'No pickle file'
continue
pd = easy_pickle.load(pkl_file)
# Check resolution limit
has_min_resolution = self.check_resolution(pd=pd)
if not has_min_resolution:
skip_data[prefix] = 'Resolution is worse than %sA: %sA' % \
(self.params.resolution_limit, pd.resolution)
continue
top_dir = os.path.join(data_dir, prefix)
currentMonth = datetime.datetime.now().month
currentYear = datetime.datetime.now().year
date_str = format(currentMonth, '02') + '_' + str(currentYear)
dest_dir = os.path.join(top_dir, date_str)
#self.dest_dir = dest_dir
#dest_dir = os.path.join(data_dir, prefix)
json_filename = os.path.join(dest_dir, pdb_code + '.json')
if (os.path.isfile(json_filename)
and self.params.overwrite is False):
with open(json_filename, 'r') as fp:
data = json.load(fp)
task_list = list(self.params.tasks)
if len(task_list) < 5:
# don't run statistics or plots if refinement was not performed
# (but it will run if failed)
if ((data['success_refinement'] is None)
and ('statistics' in self.params.tasks
or 'plots' in self.params.tasks)
and ('refinement') not in self.params.tasks):
print('Refinement not yet performed for: ',
prefix,
file=self.logger)
continue
# don't run refinement if readyset failed or was not performed
if (not data['success_readyset']
and 'refinement' in self.params.tasks):
print('Readyset not performed or failed: ',
prefix,
file=self.logger)
continue
# don't run if task already performed
if not self.params.run_again:
do_continue = False
for task_str in [
'readyset', 'refinement', 'statistics',
'plots'
]:
if task_str in self.params.tasks and data[
'success_' + task_str] is True:
msg = 'Task "%s" was already successfully performed for %s. \
\nUse keyword "run_again" to overrun previous results' % (
task_str, prefix)
print(msg, file=self.logger)
do_continue = True
break
if do_continue: continue
# if all tasks are chosen, only run if at least one failed
elif len(task_list) == 5:
_l = [data['success_' + x] for x in task_list]
if (_l.count(True) == 5):
continue
#raise Sorry(msg)
# Check if other error
if pd.error is not None:
skip_data[prefix] = pd.error
continue
# Check if PDB file exists (TODO accept also only cif instances)
pdb_filename = os.path.join(root, prefix + '.pdb')
if (os.path.isfile(pdb_filename)):
size.append(os.path.getsize(pdb_filename))
folders.append(root)
elif prefix not in skip_data.keys():
skip_data[prefix] = 'No PDB file'
#print(pd.resolution, pd.model_file, pd.map_file, pd.map_file_1, pd.map_file_2, pd.error)
# sort folders according to PDB file size, smallest are processed first
tmp = []
for i in flex.sort_permutation(size):
tmp.append(folders[i])
folders = tmp[:]
if not self.params.models:
json_filename = os.path.join(data_dir, 'skipped.json')
with open(json_filename, 'w') as jfp:
json.dump(skip_data, jfp, sort_keys=True, indent=4)
return folders
def parse_pickle(filename, out=sys.stdout):
print "===== Loading Pickle: %s =====" % filename
ringer_things = easy_pickle.load(filename)
return process_raw_results(ringer_things, out=out)
def run(self):
#
folders = list
if (self.params.mode == 'queue'):
cmd_dict = {}
for i in [1, 2, 3, 4, 6]:
cmd_dict[i] = list()
if (self.params.folder is not None):
prefix = os.path.basename(os.path.normpath(self.params.folder))
pkl_file = os.path.join(cryodir, prefix, prefix + '.pkl')
if (not os.path.isfile(pkl_file)):
print('No pickle file for ', prefix, file=self.logger)
return
pd = easy_pickle.load(pkl_file)
has_min_resolution = self.check_resolution(pd=pd)
if not has_min_resolution: return
folders = [self.params.folder]
else:
folders = self.get_folders()
#
if folders:
print('Performing tasks: ', self.params.tasks, file=self.logger)
n_jobs = 0
for folder in folders:
# -------------------------------------
if (self.params.mode == 'individual'):
# -------------------------------------
#n_jobs += 1
rr = rerefine(logger=self.logger,
folder=folder,
params=self.params)
#
if ('composition' in self.params.tasks):
rr.get_composition()
#
if ('readyset' in self.params.tasks):
rr.find_ligands_and_get_cif()
# #
if ('refinement' in self.params.tasks):
rr.run_real_space_refine()
# #
if ('statistics' in self.params.tasks):
rr.get_statistics()
rr.get_composition(run_map_box=False)
# #
if ('plots' in self.params.tasks):
rr.get_plots()
#
rr.save_results()
# -------------------------------------
if (self.params.mode == 'queue'):
# -------------------------------------
prefix = os.path.basename(folder)
pdb_filename = os.path.join(folder, prefix + '.pdb')
pdb_size = os.path.getsize(pdb_filename) / (1024.0 * 1024.0)
#
# TODO not sure what to do with humonguous models
if (pdb_size >= 50): continue
#
queue_tasks = '+'.join(self.params.tasks)
n_jobs += 1
#if n_jobs == 100: break
cmds = [
'iotbx.python', script, 'mode=individual',
'folder=%s' % folder,
'tasks=%s' % queue_tasks
]
if self.params.rsr_options:
for param in self.params.rsr_options:
cmds.append("rsr_options='" + param.rsr_param + "'")
if self.params.add_hydrogen:
cmds.append('add_hydrogen=True')
if self.params.overwrite:
cmds.append('overwrite=True')
cmd = " ".join(cmds)
if (pdb_size < 5): cmd_dict[1].append(cmd)
elif (pdb_size >= 5 and pdb_size < 10): cmd_dict[2].append(cmd)
elif (pdb_size >= 10 and pdb_size < 15):
cmd_dict[3].append(cmd)
elif (pdb_size >= 15 and pdb_size < 20):
cmd_dict[4].append(cmd)
elif (pdb_size >= 20 and pdb_size < 50):
cmd_dict[6].append(cmd)
if (self.params.mode == 'queue'):
for threads in cmd_dict:
commands = cmd_dict[threads]
params_threads = [int(i) for i in self.params.threads]
if threads not in params_threads: continue
for command in commands[:6]:
print(command)
if not self.params.machine:
machine = 'rebus'
else:
machine = self.params.machine
if threads > 1:
qsub_cmd = 'qsub -q all.q@%s -pe threaded %s' % (machine,
threads)
else:
qsub_cmd = 'qsub -q all.q@%s' % machine
currentMonth = datetime.datetime.now().month
currentYear = datetime.datetime.now().year
date_str = format(currentMonth, '02') + '_' + str(currentYear)
queue_log_dir = os.path.join(
data_dir, 'queue_logs_' + date_str,
str(datetime.date.today()) + '_' + str(threads) +
'_threads')
if (not os.path.isdir(queue_log_dir)):
os.makedirs(queue_log_dir)
else:
time_hour = str(datetime.datetime.now())[11:16]
time_hour = time_hour.replace(':', '_')
queue_log_dir = queue_log_dir + '_' + time_hour
os.makedirs(queue_log_dir)
print(queue_log_dir)
easy_qsub.run(
phenix_source=phenix_dir,
where=queue_log_dir,
commands=commands,
#qsub_cmd = 'qsub -q all.q@%s -pe threaded 4' % machine,
qsub_cmd=qsub_cmd,
#qsub_cmd = 'qsub -q all.q@morse',
js=3,
size_of_chunks=1)
def __init__(self,
pixel_histograms,
output_dirname=".",
gain_map_path=None,
gain_map=None,
method="photon_counting",
estimated_gain=30,
nproc=None,
photon_threshold=2 / 3,
roi=None,
run=None):
assert method in ("sum_adu", "photon_counting")
self.sum_img = flex.double(flex.grid(
370, 391), 0) # XXX define the image size some other way?
gain_img = flex.double(self.sum_img.accessor(), 0)
assert [gain_map, gain_map_path].count(None) > 0
if gain_map_path is not None:
d = easy_pickle.load(gain_map_path)
gain_map = d["DATA"]
two_photon_threshold = photon_threshold + 1
mask = flex.int(self.sum_img.accessor(), 0)
start_row = 370
end_row = 0
print(len(pixel_histograms.histograms))
pixels = list(pixel_histograms.pixels())
n_pixels = len(pixels)
if roi is not None:
for k, (i, j) in enumerate(reversed(pixels)):
if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
del pixels[n_pixels - k - 1]
if gain_map is None:
fixed_func = pixel_histograms.fit_one_histogram
else:
def fixed_func(pixel):
return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)
results = None
if nproc is None: nproc = easy_mp.Auto
nproc = easy_mp.get_processes(nproc)
print("nproc: ", nproc)
stdout_and_results = easy_mp.pool_map(
processes=nproc,
fixed_func=fixed_func,
args=pixels,
func_wrapper="buffer_stdout_stderr")
results = [r for so, r in stdout_and_results]
gains = flex.double()
for i, pixel in enumerate(pixels):
start_row = min(start_row, pixel[0])
end_row = max(end_row, pixel[0])
n_photons = 0
if results is None:
# i.e. not multiprocessing
try:
gaussians = pixel_histograms.fit_one_histogram(pixel)
except RuntimeError as e:
print("Error fitting pixel %s" % str(pixel))
print(str(e))
mask[pixel] = 1
continue
else:
gaussians = results[i]
hist = pixel_histograms.histograms[pixel]
if gaussians is None:
# Presumably the peak fitting failed in some way
print("Skipping pixel %s" % str(pixel))
continue
zero_peak_diff = gaussians[0].params[1]
if gain_map is None:
try:
view_pixel_histograms.check_pixel_histogram_fit(
hist, gaussians)
except view_pixel_histograms.PixelFitError as e:
print("PixelFitError:", str(pixel), str(e))
mask[pixel] = 1
continue
gain = gaussians[1].params[1] - gaussians[0].params[1]
gain_img[pixel] = gain
gain_ratio = gain / estimated_gain
else:
gain = gain_map[pixel]
if gain == 0:
print("bad gain!!!!!", pixel)
continue
gain = 30 / gain
gain_ratio = 1 / gain
gains.append(gain)
#for g in gaussians:
#sigma = abs(g.params[2])
#if sigma < 1 or sigma > 10:
#print "bad sigma!!!!!", pixel, sigma
#mask[pixel] = 1
#continue
if method == "sum_adu":
sum_adu = 0
one_photon_cutoff, two_photon_cutoff = [
(threshold * gain + zero_peak_diff)
for threshold in (photon_threshold, two_photon_threshold)
]
i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
slots = hist.slots().as_double()
slot_centers = hist.slot_centers()
slots -= gaussians[0](slot_centers)
for j in range(i_one_photon_cutoff, len(slots)):
center = slot_centers[j]
sum_adu += slots[j] * (center - zero_peak_diff) * 30 / gain
self.sum_img[pixel] = sum_adu
elif method == "photon_counting":
one_photon_cutoff, two_photon_cutoff = [
(threshold * gain + zero_peak_diff)
for threshold in (photon_threshold, two_photon_threshold)
]
i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
i_two_photon_cutoff = hist.get_i_slot(two_photon_cutoff)
slots = hist.slots()
for j in range(i_one_photon_cutoff, len(slots)):
if j == i_one_photon_cutoff:
center = hist.slot_centers()[j]
upper = center + 0.5 * hist.slot_width()
n_photons += int(
round((upper - one_photon_cutoff) /
hist.slot_width() * slots[j]))
elif j == i_two_photon_cutoff:
center = hist.slot_centers()[j]
upper = center + 0.5 * hist.slot_width()
n_photons += 2 * int(
round((upper - two_photon_cutoff) /
hist.slot_width() * slots[j]))
elif j < i_two_photon_cutoff:
n_photons += int(round(slots[j]))
else:
n_photons += 2 * int(round(slots[j]))
self.sum_img[pixel] = n_photons
stats = scitbx.math.basic_statistics(gains)
print("gain statistics:")
stats.show()
mask.set_selected(self.sum_img == 0, 1)
unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
mask.set_selected(unbound_pixel_mask > 0, 1)
bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
mask.set_selected(bad_pixel_mask > 0, 1)
for row in range(self.sum_img.all()[0]):
self.sum_img[row:row + 1, :].count(0)
spectrum_focus = self.sum_img[start_row:end_row, :]
mask_focus = mask[start_row:end_row, :]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print("Number of rows: %i" % spectrum_focus.all()[0])
print("Estimated no. photons counted: %i" % flex.sum(spectrum_focus))
print("Number of images used: %i" %
flex.sum(pixel_histograms.histograms.values()[0].slots()))
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
gain_map.as_1d().set_selected(img_sel.iselection(),
1 / gain_img.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_map,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'gain_map_')
plot_x, plot_y = xes_finalise.output_spectrum(
spectrum_focus.iround(),
mask_focus=mask_focus,
output_dirname=output_dirname,
run=run)
self.spectrum = (plot_x, plot_y)
self.spectrum_focus = spectrum_focus
xes_finalise.output_matlab_form(
spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
print(output_dirname)
def __init__(self,
path=None,
filename=None,
crystal_num=0,
remove_negative=False,
use_b=True,
scale=True,
dicti=None,
pixel_size=None):
"""
Constructor for SingleFrame object, using a cctbx.xfel integration pickle.
:param path: path to integration pickle
:param filename: the file name alone (used as a label)
:param crystal_num: if multiple lattices present, the latice number.
:param remove_negative: Boolean for removal of negative intensities
:param use_b: if True, initialise scale and B, if false, use only mean-intensity scaling.
:param dicti: optional. If a dictionairy is supplied here, will create object from that rather than attempting to read the file specified in path, filename.
:param pixel_size: the size of pixels in mm. Defaults to a MAR detector with a warning at debug level of logging.
:param scale: if False, will intialise scales to G=1, B=0.
:return: a SingleFrame object, with the following Object attributes:
Object attributes are:
- `is_polarization_corrected`: Boolean flag indicatinf if polarization correction has been applied
- `miller_array`: the cctbx.miller miller array of spot intensities.
- `mapped_predictions`: the mapped_predictions locations
- `path`: full path to the original file
- `name`: file-name, used as an identifier
- `crystal_system:
- `pg`: point group of pickle
- `uc`: Niggli unit cell as a tuple
- `orientation`: cctbx crystal_orientation object
- `total_i`: the total integrated intensity for this frame
- `xbeam`: x-location of beam centre
- `ybeam`: y-location of beam centre
- `wavelength:
- `spot_offset`: the mean offset between observed spots and predicted centroids. Only created if integration was performed using verbose_cv=True. Otherwise None.
- `minus_2B`: the gradient of the ln(i) vs. sinsqtheta_over_lambda_sq plot
- `G`: intercept of the of the ln(i) vs. sinsqtheta_over_lambda_sq plot
- `log_i`: list of log_i intensities
- `sinsqtheta_over_lambda_sq`: list of sinsqtheta_over_lambda_sq
- `wilson_err`: standard error on the fit of ln(i) vs. sinsqtheta_over_lambda_sq
- `miller_fullies`: a cctbx.miller array of fully recorded intensites.
"""
if dicti is not None:
d = dicti
else:
try:
d = easy_pickle.load(path)
except (cPickle.UnpicklingError, ValueError, EOFError, IOError):
d = {}
logger.warning(
"Could not read %s. It may not be a pickle file." % path)
if 'observations' not in d:
return
try:
if pixel_size:
self.pixel_size = pixel_size
else:
logger.debug(
"No pixel size specified, defaulting to MAR (0.079346). "
"Bad times if this is not the correct detector!")
self.pixel_size = 0.079346
# Warn on error, but continue directory traversal.
self.is_polarization_corrected = False
# Miller arrays
self.miller_array = d['observations'][crystal_num]
self.mapped_predictions = d['mapped_predictions'][crystal_num]
# Image pickle info
self.path = path
self.name = filename
# Unit cell info
self.crystal_system = self.miller_array.crystal_symmetry()\
.space_group().crystal_system()
self.pg = d['pointgroup'].replace(' ', '') # enforce consistency
self.uc = d['current_orientation'][crystal_num].unit_cell() \
.niggli_cell() \
.parameters()
self.orientation = d['current_orientation'][crystal_num]
# Agregate info
self.total_i = d['observations'][crystal_num].sum()
self.xbeam = d['xbeam']
self.ybeam = d['ybeam']
self.wavelength = d['wavelength']
self.distance = d['distance']
if 'correction_vectors' in d:
all_corrections = []
for spot in d['correction_vectors'][crystal_num]:
dta = np.sqrt(
(spot['refinedcenter'][0] - spot['obscenter'][0])**2 +
(spot['refinedcenter'][1] - spot['obscenter'][1])**2)
all_corrections.append(dta)
self.spot_offset = np.mean(all_corrections)
else:
self.spot_offset = None
if remove_negative:
self.filter_negative_intensities()
# Do polarization correction
self.polarization_correction()
self.minus_2B, self.G, self.log_i, \
self.sinsqtheta_over_lambda_sq, \
self.wilson_err = self.init_calc_wilson(use_b)
if not scale:
self.minus_2B = 0
self.G = 1
if logger.root.level < logging.DEBUG: # Extreme debug!
self.plot_wilson()
logger.debug("Extracted image {}".format(filename))
except KeyError:
logger.warning(
"Could not extract point group and unit cell from %s" % path)
self.miller_fullies = None
def easy_run_plot_multirun_stats(pickle):
from libtbx import easy_pickle
contents = easy_pickle.load(pickle)
stats_tuple, d_min, n_multiples, run_tags, run_statuses, minimalist, interactive, xsize, ysize, high_vis, title = contents
print(plot_run_stats(stats_tuple, d_min, n_multiples, run_tags=run_tags, run_statuses=run_statuses, minimalist=minimalist,
interactive=interactive, xsize=xsize, ysize=ysize, high_vis=high_vis, title=title))
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path.encode(), pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def run(args):
phil = iotbx.phil.process_command_line(args=args,
master_string=master_phil).show()
usage = \
""" %s input.experiment=experimentname input.run_num=N input.address=address
"""%libtbx.env.dispatcher_name
params = phil.work.extract()
if not os.path.exists(params.output.output_dir):
raise Sorry("Output path not found:" + params.output.output_dir)
if params.input.experiment is None or \
params.input.run_num is None or \
params.input.address is None:
raise Usage(usage)
# set up psana
if params.dispatch.events_accepted or params.dispatch.events_rejected:
assert params.input.cfg is not None
setConfigFile(params.input.cfg)
dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment,
params.input.run_num)
ds = DataSource(dataset_name)
src = Source('DetInfo(%s)' % params.input.address)
# set up multiprocessing with MPI
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
size = comm.Get_size() # size: number of processes running in this job
if params.dispatch.max_events is None:
max_events = sys.maxint
else:
max_events = params.dispatch.max_events
if params.input.dark is not None:
dark = easy_pickle.load('%s' % params.input.dark)
for run in ds.runs():
times = run.times()
if (params.dispatch.events_begin is None
and params.dispatch.events_end is None):
times = times[:]
elif (params.dispatch.events_begin is not None
and params.dispatch.events_end is None):
times = times[params.dispatch.events_begin:]
elif (params.dispatch.events_begin is None
and params.dispatch.events_end is not None):
times = times[:params.dispatch.events_end]
elif (params.dispatch.events_begin is not None
and params.dispatch.events_end is not None):
times = times[params.dispatch.events_begin:params.dispatch.
events_end]
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in range(nevents) if (i + rank) % size == 0]
print len(mytimes)
#mytimes = mytimes[len(mytimes)-1000:len(mytimes)]
totals = np.array([0.0])
print "initial totals", totals
for i, t in enumerate(mytimes):
print "Event", i, "of", len(mytimes),
evt = run.event(t)
if params.dispatch.events_accepted or params.dispatch.events_all:
if evt.get("skip_event") == True:
continue
elif params.dispatch.events_rejected:
if evt.get("skip_event") == False:
continue
try:
data = evt.get(Camera.FrameV1, src)
except ValueError as e:
src = Source('BldInfo(%s)' % params.input.address)
data = evt.get(Bld.BldDataSpectrometerV1, src)
if data is None:
print "No data"
continue
#set default to determine FEE data type
two_D = False
#check attribute of data for type
try:
data = np.array(data.data16().astype(np.int32))
two_D = True
except AttributeError as e:
data = np.array(data.hproj().astype(np.float64))
if two_D:
if 'dark' in locals():
data = data - dark
one_D_data = np.sum(data, 0) / data.shape[0]
two_D_data = np.double(data)
else:
#used to fix underflow problem that was present in earlier release of psana and pressent for LH80
for i in range(len(data)):
if data[i] > 1000000000:
data[i] = data[i] - (2**32)
if 'dark' in locals():
data = data - dark
one_D_data = data
totals[0] += 1
print "total good:", totals[0]
if not 'fee_one_D' in locals():
fee_one_D = one_D_data
else:
fee_one_D += one_D_data
if ('two_D_data' in locals() and not 'fee_two_D' in locals()):
fee_two_D = two_D_data
elif 'fee_two_D' in locals():
fee_two_D += two_D_data
acceptedtotals = np.zeros(totals.shape)
acceptedfee1 = np.zeros((fee_one_D.shape))
if 'fee_two_D' in locals():
acceptedfee2 = np.zeros((fee_two_D.shape))
print "Synchronizing rank", rank
comm.Reduce(fee_one_D, acceptedfee1)
comm.Reduce(totals, acceptedtotals)
if 'acceptedfee2' in locals():
comm.Reduce(fee_two_D, acceptedfee2)
print "number averaged", acceptedtotals[0]
if rank == 0:
if acceptedtotals[0] > 0:
acceptedfee1 /= acceptedtotals[0]
if 'acceptedfee2' in locals():
acceptedfee2 /= acceptedtotals[0]
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from pylab import savefig, close
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt
from matplotlib import cm
if params.dispatch.events_accepted:
easy_pickle.dump(
os.path.join(
params.output.output_dir, "fee_avg_1_D_" +
'r%s' % params.input.run_num + "_accepted.pickle"),
acceptedfee1)
pp1 = PdfPages(
os.path.join(
params.output.output_dir, "fee_avg_1_D_" +
'r%s' % params.input.run_num + "_accepted.pdf"))
if 'acceptedfee2' in locals():
easy_pickle.dump(
os.path.join(
params.output.output_dir, "fee_avg_2_D_" +
'r%s' % params.input.run_num + "_accepted.pickle"),
acceptedfee2)
pp2 = PdfPages(
os.path.join(
params.output.output_dir, "fee_avg_2_D_" +
'r%s' % params.input.run_num + "_accepted.pdf"))
if params.dispatch.events_all:
easy_pickle.dump(
os.path.join(
params.output.output_dir, "fee_avg_1_D_" +
'r%s' % params.input.run_num + "_all.pickle"),
acceptedfee1)
pp1 = PdfPages(
os.path.join(
params.output.output_dir, "fee_avg_1_D_" +
'r%s' % params.input.run_num + "_all.pdf"))
if 'acceptedfee2' in locals():
easy_pickle.dump(
os.path.join(
params.output.output_dir, "fee_avg_2_D_" +
'r%s' % params.input.run_num + "_all.pickle"),
acceptedfee2)
pp2 = PdfPages(
os.path.join(
params.output.output_dir, "fee_avg_2_D_" +
'r%s' % params.input.run_num + "_all.pdf"))
if params.dispatch.events_rejected:
easy_pickle.dump(
os.path.join(
params.output.output_dir, "fee_avg_1_D_" +
'r%s' % params.input.run_num + "_rejected.pickle"),
acceptedfee1)
pp1 = PdfPages(
os.path.join(
params.output.output_dir, "fee_avg_1_D_" +
'r%s' % params.input.run_num + "_rejected.pdf"))
if 'acceptedfee2' in locals():
easy_pickle.dump(
os.path.join(
params.output.output_dir, "fee_avg_2_D_" +
'r%s' % params.input.run_num + "_rejected.pickle"),
acceptedfee2)
pp2 = PdfPages(
os.path.join(
params.output.output_dir, "fee_avg_2_D_" +
'r%s' % params.input.run_num + "_rejected.pdf"))
print "Done"
#plotting result
# matplotlib needs a different backend when run on the cluster nodes at SLAC
# these two lines not needed when working interactively at SLAC, or on mac or on viper
if params.input.pixel_to_eV.energy_per_px is not None:
xvals = (
np.array(range(acceptedfee1.shape[0])) -
params.input.pixel_to_eV.x_coord_one
) * params.input.pixel_to_eV.energy_per_px + params.input.pixel_to_eV.y_coord_one
xvals = xvals[::-1]
if params.input.pixel_to_eV.x_coord_two is not None:
eV_per_px = (params.input.pixel_to_eV.y_coord_two -
params.input.pixel_to_eV.y_coord_one) / (
params.input.pixel_to_eV.x_coord_two -
params.input.pixel_to_eV.x_coord_one)
xvals = (np.array(range(acceptedfee1.shape[0])) -
params.input.pixel_to_eV.x_coord_one
) * eV_per_px + params.input.pixel_to_eV.y_coord_one
xvals = xvals[::-1]
if params.input.pixel_to_eV.x_coord_two is None and params.input.pixel_to_eV.energy_per_px is None:
xvals = np.arange(0, len(acceptedfee1), 1)
yvals = acceptedfee1
def OneD_plot(X, Y):
plt.figure()
plt.clf()
plt.plot(X, Y)
if params.dispatch.events_accepted:
plt.title('Accepted Shots FEE Spectrum Run %s' %
params.input.run_num)
elif params.dispatch.events_all:
plt.title('All Shots FEE Spectrum Run %s' %
params.input.run_num)
elif params.dispatch.events_rejected:
plt.title('Rejected Shots FEE Spectrum Run %s' %
params.input.run_num)
if params.input.pixel_to_eV.x_coord_one is not None:
plt.xlabel('eV', fontsize=13)
else:
plt.xlabel('pixels', fontsize=13)
plt.ylabel('pixels', fontsize=13)
pp1.savefig()
def TwoD_plot(data):
plt.figure()
ax = plt.gca()
# use specified range 0, 50 to plot runs 117 - 201
#min=0, vmax=50
cax = ax.imshow(data,
interpolation='nearest',
origin='lower',
cmap=cm.coolwarm)
plt.colorbar(cax, fraction=0.014, pad=0.04)
if params.dispatch.events_accepted:
ax.set_title('Accepted 2-D FEE Spectrum Run %s' %
params.input.run_num)
elif params.dispatch.events_all:
ax.set_title('All 2-D FEE Spectrum Run %s' %
params.input.run_num)
elif params.dispatch.events_rejected:
ax.set_title('Rejected 2-D FEE Spectrum Run %s' %
params.input.run_num)
pp2.savefig()
OneD_plot(xvals, yvals)
pp1.close()
if 'acceptedfee2' in locals():
TwoD_plot(acceptedfee2)
pp2.close()
def run(self):
from os.path import join, exists
from libtbx import easy_run
assert(exists(join(self.path, "datablock.json")))
input_filename = join(self.path, "datablock.json")
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
]).raise_if_errors()
assert(exists("mask.pickle"))
print 'OK'
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
'output.mask=mask2.pickle',
'output.datablock=masked_datablock.json',
'untrusted.rectangle=100,200,100,200'
]).raise_if_errors().show_stdout()
assert(exists("mask2.pickle"))
assert(exists("masked_datablock.json"))
from dxtbx.serialize import load
datablocks = load.datablock("masked_datablock.json")
imageset = datablocks[0].extract_imagesets()[0]
import os
assert imageset.external_lookup.mask.filename == os.path.join(
os.path.abspath(os.getcwd()), 'mask2.pickle')
print 'OK'
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
'output.mask=mask3.pickle',
'untrusted.circle=100,100,10'
]).raise_if_errors()
assert(exists("mask3.pickle"))
print 'OK'
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
'output.mask=mask4.pickle',
'resolution_range=2,3',
]).raise_if_errors()
assert(exists("mask4.pickle"))
print 'OK'
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
'output.mask=mask5.pickle',
'd_min=3',
'd_max=2',
]).raise_if_errors()
assert(exists("mask5.pickle"))
print 'OK'
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
'output.mask=mask6.pickle',
'\'ice_rings{filter=True;d_min=2}\'',
]).raise_if_errors()
assert(exists("mask6.pickle"))
print 'OK'
# Call dials.generate_mask
easy_run.fully_buffered([
'dials.generate_mask',
input_filename,
'output.mask=mask3.pickle',
'untrusted.polygon=100,100,100,200,200,200,200,100',
'untrusted.pixel=0,0',
'untrusted.pixel=1,1'
]).raise_if_errors()
assert(exists("mask3.pickle"))
from libtbx import easy_pickle
mask = easy_pickle.load("mask3.pickle")
assert not mask[0][0,0]
assert not mask[0][1,1]
assert mask[0][0,1]
print 'OK'
def run():
if not have_dials_regression:
print "Skipping tst_reindex: dials_regression not available."
return
data_dir = os.path.join(dials_regression, "indexing_test_data",
"i04_weak_data")
pickle_path = os.path.join(data_dir, "indexed.pickle")
experiments_path = os.path.join(data_dir, "experiments.json")
commands = [
"dials.reindex", pickle_path, experiments_path,
"change_of_basis_op=2a,b,c", "space_group=P1"
]
command = " ".join(commands)
print command
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory()
os.chdir(tmp_dir)
result = easy_run.fully_buffered(command=command).raise_if_errors()
old_reflections = easy_pickle.load(pickle_path)
assert os.path.exists('reindexed_reflections.pickle')
new_reflections = easy_pickle.load('reindexed_reflections.pickle')
old_experiments = load.experiment_list(experiments_path,
check_format=False)
assert os.path.exists('reindexed_experiments.json')
new_experiments = load.experiment_list('reindexed_experiments.json',
check_format=False)
h1, k1, l1 = old_reflections['miller_index'].as_vec3_double().parts()
h2, k2, l2 = new_reflections['miller_index'].as_vec3_double().parts()
assert approx_equal(2 * h1, h2)
assert approx_equal(k1, k2)
assert approx_equal(l1, l2)
old_uc_params = old_experiments[0].crystal.get_unit_cell().parameters()
new_uc_params = new_experiments[0].crystal.get_unit_cell().parameters()
assert approx_equal(2 * old_uc_params[0], new_uc_params[0])
assert approx_equal(old_uc_params[1:], new_uc_params[1:])
assert old_experiments[0].crystal.get_space_group().type().hall_symbol(
) == ' P 1'
assert new_experiments[0].crystal.get_space_group().type().hall_symbol(
) == ' P 1'
# set space group P4
from cctbx import sgtbx
cb_op = sgtbx.change_of_basis_op('a,b,c')
commands = [
"dials.reindex", experiments_path, "space_group=P4",
"change_of_basis_op=%s" % str(cb_op), "output.experiments=P4.json"
]
command = " ".join(commands)
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
# apply one of the symops from the space group
cb_op = sgtbx.change_of_basis_op('-x,-y,z')
commands = [
"dials.reindex", "P4.json",
"change_of_basis_op=%s" % str(cb_op),
"output.experiments=P4_reindexed.json"
]
command = " ".join(commands)
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
new_experiments1 = load.experiment_list('P4_reindexed.json',
check_format=False)
assert approx_equal(new_experiments1[0].crystal.get_A(),
old_experiments[0].crystal.change_basis(cb_op).get_A())
#
cb_op = sgtbx.change_of_basis_op('-x,-y,z')
commands = [
"dials.reindex", "P4.json", "change_of_basis_op=auto",
"reference=P4_reindexed.json", "output.experiments=P4_reindexed2.json"
]
command = " ".join(commands)
print command
result = easy_run.fully_buffered(command=command).raise_if_errors()
new_experiments2 = load.experiment_list('P4_reindexed2.json',
check_format=False)
assert approx_equal(new_experiments1[0].crystal.get_A(),
new_experiments2[0].crystal.get_A())
def cv_png(image_pickle,
integration_pickle,
file_name=None,
res=600,
show_spots=True):
""" Write a png file visualizing the correction vectors.
:param image_pickle: path to image pickle file.
:param integration_pickle: path to integration pickle file.
:param res: resolution of output file in dpi (6x6 image size).
:file_name: desired output file name. Defaults to the integration_pickle name.
"""
cmap = mpl.cm.jet
if file_name is None:
import os
# Change extension of `image_pickle` to .png
file_name = os.path.splitext(integration_pickle)[0] + ".png"
# Load image pickle, and convert to image
img_dict = ep.load(image_pickle)
img_data = img_dict['DATA'].as_numpy_array()
# Load integration pickle, and get coordinates of predictions
int_d = ep.load(integration_pickle)
spot_coords = [x['obsspot'] for x in int_d['correction_vectors'][0]]
spot_coords = np.array(spot_coords).transpose()
pred_coords = [x['predspot'] for x in int_d['correction_vectors'][0]]
pred_coords = np.array(pred_coords).transpose()
corr_vecs = pred_coords - spot_coords
corr_factors = np.sqrt(abs(corr_vecs[0]**2 - corr_vecs[1]**2))
# Get some other useful info from integration pickle
point_group = int_d['pointgroup']
unit_cell = int_d['current_orientation'][0].unit_cell().parameters()
unit_cell = ', '.join("{:.1f}".format(u) for u in unit_cell)
mosaicity = int_d['mosaicity']
# Create the figure
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(6, 6))
# 1st set of axes for the image
ax = fig.add_subplot(1, 1, 1)
plt.subplots_adjust(right=0.8)
ax.set_xlim(0, len(img_data[1]))
ax.set_ylim(0, len(img_data[0]))
ax.set_aspect('equal')
# To display image properly, need to set img limit to 0.01th percentile
# a few maxed-out pixels will make the entire image appear blank white
clim = (img_data.min(), np.percentile(img_data, 99.99))
ax.imshow(img_data, origin=None, cmap='Greys', clim=clim)
# 2nd set of axes for the predictions
ax2 = fig.add_axes(ax.get_position(), frameon=False) # superimposed axes
ax2.set_xlim(0, len(img_data[1]))
ax2.set_ylim(0, len(img_data[0]))
ax2.set_aspect('equal')
norm = mpl.colors.Normalize(vmin=0, vmax=round(max(corr_factors)))
ax2.scatter(spot_coords[1],
spot_coords[0],
c=norm(corr_factors),
cmap=cmap,
alpha=0.5,
s=5,
linewidths=0)
cax = plt.axes([0.85, 0.15, 0.05, 0.69])
cbar = mpl.colorbar.ColorbarBase(cax, cmap=cmap, norm=norm)
cbar.set_label('Spot-prediction distance [px]')
ax.set_title("Prediction-offset values\nfor spotfinder results")
plt.savefig(file_name, dpi=res, format='png')
