def _launch(self):
"""
Usually this method should be modified when new features are added to refinement
"""
# TODO return None or refiner instance
LOGGER.info("begin _launch")
x_init = None
nmacro = self.params.refiner.num_macro_cycles
n_trials = len(self.params.refiner.max_calls)
for i_trial in range(n_trials * nmacro):
self.RUC = StageTwoRefiner(self.Modelers, self.symbol, self.params)
if self.will_refine(self.params.refiner.refine_spot_scale):
self.RUC.refine_crystal_scale = (
self.params.refiner.refine_spot_scale * nmacro)[i_trial]
if self.will_refine(self.params.refiner.refine_Fcell):
self.RUC.refine_Fcell = (self.params.refiner.refine_Fcell *
nmacro)[i_trial]
self.RUC.panel_group_from_id = self.panel_group_from_id
self.RUC.panel_reference_from_id = self.panel_reference_from_id
self.RUC.panel_groups_being_refined = self.panel_groups_refined
# TODO verify not refining Fcell in case of local refiner
self.RUC.max_calls = (self.params.refiner.max_calls *
nmacro)[i_trial]
self.RUC.x_init = x_init
self.RUC.ignore_line_search_failed_step_at_lower_bound = True # TODO: why was this necessary?
# plot things
self.RUC.trial_id = i_trial
self.RUC.log_fcells = True
self.RUC.request_diag_once = False
self.RUC.trad_conv = True
self.RUC.idx_from_asu = self.idx_from_asu
self.RUC.asu_from_idx = self.asu_from_idx
self.RUC.S = self.SIM
self.RUC.restart_file = self.params.refiner.io.restart_file
self.RUC.S.update_nanoBragg_instance(
'update_oversample_during_refinement',
self.params.refiner.update_oversample_during_refinement)
self.RUC.S.update_nanoBragg_instance("Npix_to_allocate",
self.NPIX_TO_ALLOC)
self.RUC.S.update_nanoBragg_instance('device_Id', self.DEVICE_ID)
self.RUC.use_curvatures_threshold = self.params.refiner.use_curvatures_threshold
if not self.params.refiner.curvatures:
self.RUC.S.update_nanoBragg_instance('compute_curvatures',
False)
if COMM.rank == 0:
self.RUC.S.update_nanoBragg_instance(
'verbose', self.params.refiner.verbose)
LOGGER.info("_launch run setup")
self.RUC.run(setup_only=True)
LOGGER.info("_launch done run setup")
# for debug purposes:
#if not self.params.refiner.quiet:
# print("\n<><><><><><><><>TRIAL %d refinement status:" % i_trial)
# self.RUC.S.D.print_if_refining()
self.RUC.num_positive_curvatures = 0
self.RUC.use_curvatures = self.params.refiner.start_with_curvatures
self.RUC.hit_break_to_use_curvatures = False
# selection flags set here:
#self.RUC.selection_flags = self.shot_selection_flags
#if self.params.refiner.res_ranges is not None:
# assert self.shot_reso is not None, "cant set reso flags is rlp is not in refl tables"
# nshots = len(self.shot_selection_flags)
# more_sel_flags = {}
# res_ranges = utils.parse_reso_string(self.params.refiner.res_ranges)
# for i_shot in range(nshots):
# rhigh, rlow = (res_ranges*nmacro)[i_trial]
# sel_flags = self.shot_selection_flags[i_shot]
# res_flags = [rhigh < r < rlow for r in self.shot_reso[i_shot]]
# more_sel_flags[i_shot] = [flag1 and flag2 for flag1,flag2 in zip(sel_flags, res_flags)]
# self.RUC.selection_flags = more_sel_flags
LOGGER.info("_launcher runno setup")
self.RUC.run(setup=False)
LOGGER.info("_launcher done runno setup")
if self.RUC.hit_break_to_use_curvatures:
self.RUC.fix_params_with_negative_curvature = False
self.RUC.num_positive_curvatures = 0
self.RUC.use_curvatures = True
self.RUC.run(setup=False)
if self.RUC.hit_break_signal:
if self.params.profile:
self.RUC.S.D.show_timings(self.RUC.rank)
self.RUC._MPI_barrier()
break
if self.params.refiner.debug_pixel_panelfastslow is not None:
utils.show_diffBragg_state(
self.RUC.S.D,
self.params.refiner.debug_pixel_panelfastslow)
s = self.RUC._get_spot_scale(0)
print("refiner spot scale=%f" % (s**2))
x_init = self.RUC.x
if self.params.profile:
self.RUC.S.D.show_timings(self.RUC.rank)
if os.environ.get("DIFFBRAGG_USE_CUDA") is not None:
self.RUC.S.D.gpu_free()
def save_up(Modeler, x, exp, i_exp, input_refls):
LOGGER = logging.getLogger("refine")
Modeler.best_model, _ = hopper_utils.model(x, Modeler.SIM, Modeler.pan_fast_slow, compute_grad=False)
Modeler.best_model_includes_background = False
LOGGER.info("Optimized values for i_exp %d:" % i_exp)
hopper_utils.look_at_x(x, Modeler.SIM)
rank_imgs_outdir = make_rank_outdir(Modeler.params.outdir, "imgs")
rank_SIMlog_outdir = make_rank_outdir(Modeler.params.outdir, "simulator_state")
rank_refls_outdir = make_rank_outdir(Modeler.params.outdir, "refls")
rank_spectra_outdir = make_rank_outdir(Modeler.params.outdir, "spectra")
basename = os.path.splitext(os.path.basename(exp))[0]
img_path = os.path.join(rank_imgs_outdir, "%s_%s_%d.h5" % (Modeler.params.tag, basename, i_exp))
SIMlog_path = os.path.join(rank_SIMlog_outdir, "%s_%s_%d.txt" % (Modeler.params.tag, basename, i_exp))
new_refls_file = os.path.join(rank_refls_outdir, "%s_%s_%d.refl" % (Modeler.params.tag, basename, i_exp))
spectra_path = os.path.join(rank_spectra_outdir, "%s_%s_%d.lam" % (Modeler.params.tag, basename, i_exp))
if Modeler.SIM.num_xtals == 1:
save_to_pandas(x, Modeler.SIM, exp, Modeler.params, Modeler.E, i_exp, input_refls, img_path)
data_subimg, model_subimg, trusted_subimg, bragg_subimg = Modeler.get_data_model_pairs()
wavelen_subimg = []
if Modeler.SIM.D.store_ave_wavelength_image:
bm = Modeler.best_model.copy()
Modeler.best_model = Modeler.SIM.D.ave_wavelength_image().as_numpy_array()
Modeler.best_model_includes_background = True
_, wavelen_subimg, _,_ = Modeler.get_data_model_pairs()
Modeler.best_model = bm
Modeler.best_model_includes_background = False
comp = {"compression": "lzf"}
new_refls = deepcopy(Modeler.refls)
has_xyzcal = 'xyzcal.px' in list(new_refls.keys())
if has_xyzcal:
new_refls['dials.xyzcal.px'] = deepcopy(new_refls['xyzcal.px'])
new_xycalcs = flex.vec3_double(len(Modeler.refls), (np.nan,np.nan,np.nan))
h5_roi_id = flex.int(len(Modeler.refls), -1)
with h5py.File(img_path, "w") as h5:
sigmaZs = []
for i_roi in range(len(data_subimg)):
dat = data_subimg[i_roi]
fit = model_subimg[i_roi]
trust = trusted_subimg[i_roi]
sig = np.sqrt(fit + Modeler.sigma_rdout**2)
Z = (dat - fit) / sig
sigmaZ = Z[trust].std()
sigmaZs.append(sigmaZ)
h5.create_dataset("data/roi%d" % i_roi, data=data_subimg[i_roi], **comp)
h5.create_dataset("model/roi%d" % i_roi, data=model_subimg[i_roi], **comp)
if wavelen_subimg:
h5.create_dataset("wavelength/roi%d" % i_roi, data=wavelen_subimg[i_roi], **comp)
if bragg_subimg[0] is not None:
h5.create_dataset("bragg/roi%d" % i_roi, data=bragg_subimg[i_roi], **comp)
if np.any(bragg_subimg[i_roi] > 0):
I = bragg_subimg[i_roi]
Y,X = np.indices(bragg_subimg[i_roi].shape)
x1,_,y1,_ = Modeler.rois[i_roi]
X += x1
Y += y1
Isum = I.sum()
xcom = (X*I).sum() / Isum
ycom = (Y*I).sum() / Isum
com = xcom+.5, ycom+.5, 0
ref_idx = Modeler.refls_idx[i_roi]
h5_roi_id[ref_idx] = i_roi
new_xycalcs[ref_idx] = com
h5.create_dataset("rois", data=Modeler.rois)
h5.create_dataset("pids", data=Modeler.pids)
h5.create_dataset("sigma_rdout", data=Modeler.sigma_rdout)
h5.create_dataset("sigmaZ_vals", data=sigmaZs)
if Modeler.Hi_asu is not None:
h5.create_dataset("Hi_asu", data=Modeler.Hi_asu)
new_refls["xyzcal.px"] = new_xycalcs
new_refls["h5_roi_idx"] = h5_roi_id
if Modeler.params.filter_unpredicted_refls_in_output:
sel = [not np.isnan(x) for x,y,z in new_xycalcs]
new_refls = new_refls.select(flex.bool(sel))
new_refls.as_file(new_refls_file)
if Modeler.params.refiner.debug_pixel_panelfastslow is not None:
# TODO separate diffBragg logger
utils.show_diffBragg_state(Modeler.SIM.D, Modeler.params.refiner.debug_pixel_panelfastslow)
print("Refiner scale=%f" % Modeler.SIM.Scale_params[0].get_val(x[0]))
hopper_utils.finalize_SIM(Modeler.SIM, log=SIMlog_path, lam=spectra_path)
def save_up(Modeler, x, exp, i_exp, input_refls):
LOGGER = logging.getLogger("refine")
Modeler.best_model, _ = hopper_utils.model(x,
Modeler.SIM,
Modeler.pan_fast_slow,
compute_grad=False)
Modeler.best_model_includes_background = False
LOGGER.info("Optimized values for i_exp %d:" % i_exp)
hopper_utils.look_at_x(x, Modeler.SIM)
rank_imgs_outdir = make_rank_outdir(Modeler.params.outdir, "imgs")
rank_SIMlog_outdir = make_rank_outdir(Modeler.params.outdir,
"simulator_state")
rank_refls_outdir = make_rank_outdir(Modeler.params.outdir, "refls")
rank_spectra_outdir = make_rank_outdir(Modeler.params.outdir, "spectra")
basename = os.path.splitext(os.path.basename(exp))[0]
img_path = os.path.join(
rank_imgs_outdir,
"%s_%s_%d.h5" % (Modeler.params.tag, basename, i_exp))
SIMlog_path = os.path.join(
rank_SIMlog_outdir,
"%s_%s_%d.txt" % (Modeler.params.tag, basename, i_exp))
new_refls_file = os.path.join(
rank_refls_outdir,
"%s_%s_%d.refl" % (Modeler.params.tag, basename, i_exp))
spectra_path = os.path.join(
rank_spectra_outdir,
"%s_%s_%d.lam" % (Modeler.params.tag, basename, i_exp))
if Modeler.SIM.num_xtals == 1:
save_to_pandas(x, Modeler.SIM, exp, Modeler.params, Modeler.E, i_exp,
input_refls, img_path)
if isinstance(Modeler.all_sigma_rdout, np.ndarray):
data_subimg, model_subimg, trusted_subimg, bragg_subimg, sigma_rdout_subimg = Modeler.get_data_model_pairs(
)
else:
data_subimg, model_subimg, trusted_subimg, bragg_subimg = Modeler.get_data_model_pairs(
)
sigma_rdout_subimg = None
wavelen_subimg = []
if Modeler.SIM.D.store_ave_wavelength_image:
bm = Modeler.best_model.copy()
Modeler.best_model = Modeler.SIM.D.ave_wavelength_image(
).as_numpy_array()
Modeler.best_model_includes_background = True
_, wavelen_subimg, _, _ = Modeler.get_data_model_pairs()
Modeler.best_model = bm
Modeler.best_model_includes_background = False
comp = {"compression": "lzf"}
new_refls = deepcopy(Modeler.refls)
has_xyzcal = 'xyzcal.px' in list(new_refls.keys())
if has_xyzcal:
new_refls['dials.xyzcal.px'] = deepcopy(new_refls['xyzcal.px'])
per_refl_scales = flex.double(len(new_refls), 1)
new_xycalcs = flex.vec3_double(len(Modeler.refls),
(np.nan, np.nan, np.nan))
h5_roi_id = flex.int(len(Modeler.refls), -1)
with h5py.File(img_path, "w") as h5:
sigmaZs = []
for i_roi in range(len(data_subimg)):
dat = data_subimg[i_roi]
fit = model_subimg[i_roi]
trust = trusted_subimg[i_roi]
if sigma_rdout_subimg is not None:
sig = np.sqrt(fit + sigma_rdout_subimg[i_roi]**2)
else:
sig = np.sqrt(fit + Modeler.nominal_sigma_rdout**2)
Z = (dat - fit) / sig
sigmaZ = Z[trust].std()
sigmaZs.append(sigmaZ)
h5.create_dataset("data/roi%d" % i_roi,
data=data_subimg[i_roi],
**comp)
h5.create_dataset("model/roi%d" % i_roi,
data=model_subimg[i_roi],
**comp)
if wavelen_subimg:
h5.create_dataset("wavelength/roi%d" % i_roi,
data=wavelen_subimg[i_roi],
**comp)
if bragg_subimg[0] is not None:
h5.create_dataset("bragg/roi%d" % i_roi,
data=bragg_subimg[i_roi],
**comp)
if np.any(bragg_subimg[i_roi] > 0):
ref_idx = Modeler.refls_idx[i_roi]
ref = Modeler.refls[ref_idx]
I = bragg_subimg[i_roi]
Y, X = np.indices(bragg_subimg[i_roi].shape)
x1, x2, y1, y2 = Modeler.rois[i_roi]
com_x, com_y, _ = ref["xyzobs.px.value"]
com_x = int(com_x - x1 - 0.5)
com_y = int(com_y - y1 - 0.5)
# make sure at least some signal is at the centroid! otherwise this is likely a neighboring spot
try:
if I[com_y, com_x] == 0:
continue
except IndexError:
continue
X += x1
Y += y1
Isum = I.sum()
xcom = (X * I).sum() / Isum
ycom = (Y * I).sum() / Isum
com = xcom + .5, ycom + .5, 0
h5_roi_id[ref_idx] = i_roi
new_xycalcs[ref_idx] = com
scale_p = Modeler.SIM.P["scale_roi%d" % i_roi]
per_refl_scales[ref_idx] = scale_p.get_val(x[scale_p.xpos])
h5.create_dataset("rois", data=Modeler.rois)
h5.create_dataset("pids", data=Modeler.pids)
h5.create_dataset("sigma_rdout", data=Modeler.all_sigma_rdout)
h5.create_dataset("sigmaZ_vals", data=sigmaZs)
if Modeler.Hi_asu is not None:
h5.create_dataset("Hi_asu", data=Modeler.Hi_asu)
new_refls["xyzcal.px"] = new_xycalcs
new_refls["scale_factor"] = per_refl_scales
new_refls["h5_roi_idx"] = h5_roi_id
if Modeler.params.filter_unpredicted_refls_in_output:
sel = [not np.isnan(x) for x, y, z in new_xycalcs]
new_refls = new_refls.select(flex.bool(sel))
new_refls.as_file(new_refls_file)
if True:
modeler_file = os.path.join(
rank_imgs_outdir,
"%s_%s_%d_modeler.npy" % (Modeler.params.tag, basename, i_exp))
np.save(modeler_file, Modeler)
spectrum_file = os.path.join(
rank_imgs_outdir,
"%s_%s_%d_spectra.lam" % (Modeler.params.tag, basename, i_exp))
hopper_utils.write_SIM_logs(Modeler.SIM, lam=spectrum_file)
if Modeler.params.refiner.debug_pixel_panelfastslow is not None:
# TODO separate diffBragg logger
utils.show_diffBragg_state(
Modeler.SIM.D, Modeler.params.refiner.debug_pixel_panelfastslow)
print("Refiner scale=%f" % Modeler.SIM.Scale_params[0].get_val(x[0]))
hopper_utils.finalize_SIM(Modeler.SIM, log=SIMlog_path, lam=spectra_path)