def compute_functional_and_gradients(O): O.number_of_function_evaluations += 1 O.unpack_parameters() try: f_calc = O.f_obs.structure_factors_from_scatterers( xray_structure=O.xray_structure, algorithm="direct", cos_sin_table=False).f_calc() except RuntimeError as e: print("RuntimeError f_calc:", e) sys.stdout.flush() raise RuntimeError("f_calc") ls = xray.targets_least_squares(compute_scale_using_all_data=False, obs_type="F", obs=O.f_obs.data(), weights=O.weights, r_free_flags=None, f_calc=f_calc.data(), derivatives_depth=2, scale_factor=1) gact = O.xray_structure.grads_and_curvs_target_simple( miller_indices=O.f_obs.indices(), da_db=ls.gradients_work(), daa_dbb_dab=ls.hessians_work()) g_all = gact.grads g_active = flex.double() i_all = 0 sstab = O.xray_structure.site_symmetry_table() for i_sc, sc in enumerate(O.xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 else: np = site_symmetry.site_constraints().n_independent_params() g_active.extend(g_all[i_all:i_all + np + 1]) np += 4 # u_iso, occ, fp, fdp i_all += np assert i_all == g_all.size() assert g_active.size() == O.x.size() O.f_last = ls.target_work() O.g_last = g_active / O.p_as_x return O.f_last, O.g_last
def e_pot(O, sites_moved): if (O.last_sites_moved is None or O.last_sites_moved.id() != sites_moved.id()): O.last_sites_moved = sites_moved xs = O.xray_structure assert len(sites_moved) == xs.scatterers().size() xs.set_sites_cart(sites_cart=sites_moved) f_calc = O.f_obs.structure_factors_from_scatterers( xray_structure=xs, algorithm="direct", cos_sin_table=False).f_calc() from cctbx import xray tg = xray.targets_least_squares(compute_scale_using_all_data=True, obs_type="F", obs=O.f_obs.data(), weights=None, r_free_flags=None, f_calc=f_calc.data(), derivatives_depth=0, scale_factor=0) O.f = tg.target_work() return O.f
def e_pot(O, sites_moved): if ( O.last_sites_moved is None or O.last_sites_moved.id() != sites_moved.id()): O.last_sites_moved = sites_moved xs = O.xray_structure assert len(sites_moved) == xs.scatterers().size() xs.set_sites_cart(sites_cart=sites_moved) f_calc = O.f_obs.structure_factors_from_scatterers( xray_structure=xs, algorithm="direct", cos_sin_table=False).f_calc() from cctbx import xray tg = xray.targets_least_squares( compute_scale_using_all_data=True, obs_type="F", obs=O.f_obs.data(), weights=None, r_free_flags=None, f_calc=f_calc.data(), derivatives_depth=0, scale_factor=0) O.f = tg.target_work() return O.f
def get_curvs_work(f_obs, weights, xray_structure, lim_eps): f_calc = f_obs.structure_factors_from_scatterers( xray_structure=xray_structure, algorithm="direct", cos_sin_table=False).f_calc() ls = xray.targets_least_squares(compute_scale_using_all_data=False, obs_type="F", obs=abs(f_calc).data(), weights=weights, r_free_flags=None, f_calc=f_calc.data(), derivatives_depth=2, scale_factor=1) gact = xray_structure.grads_and_curvs_target_simple( miller_indices=f_obs.indices(), da_db=ls.gradients_work(), daa_dbb_dab=ls.hessians_work()) c_all = gact.curvs c_active_site = flex.double() c_active_u_iso = flex.double() i_all = 0 sstab = xray_structure.site_symmetry_table() for i_sc, sc in enumerate(xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 else: np = site_symmetry.site_constraints().n_independent_params() c_active_site.extend(c_all[i_all:i_all + np]) c_active_u_iso.append(c_all[i_all + np]) np += 4 # u_iso, occ, fp, fdp i_all += np assert i_all == c_all.size() # cf_site = curv_filter(curvs=c_active_site, lim_eps=lim_eps) cf_u_iso = curv_filter(curvs=c_active_u_iso, lim_eps=lim_eps) # result = flex.double() i_all = 0 sstab = xray_structure.site_symmetry_table() for i_sc, sc in enumerate(xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 else: np = site_symmetry.site_constraints().n_independent_params() result.extend(cf_site.apply(c_all[i_all:i_all + np])) result.extend(cf_u_iso.apply(c_all[i_all + np:i_all + np + 1])) np += 4 # u_iso, occ, fp, fdp i_all += np assert i_all == c_all.size() print "curv site below limit: %d of %d" % ( cf_site.n_below_limit, cf_site.n_below_limit + cf_site.n_above_limit) print "curv u_iso below limit: %d of %d" % (cf_u_iso.n_below_limit, cf_u_iso.n_below_limit + cf_u_iso.n_above_limit) sys.stdout.flush() return result
class ls_refinement(object): def __init__(O, f_obs, xray_structure, params, lbfgs_termination_params=None, lbfgs_exception_handling_params=None, reference_structure=None): O.f_obs = f_obs O.weights = flex.double(f_obs.data().size(), 1) O.xray_structure = xray_structure O.params = params O.reference_structure = reference_structure O.curvs_work = get_curvs_work(f_obs=O.f_obs, weights=O.weights, xray_structure=O.xray_structure, lim_eps=O.params.curv_filter_lim_eps) if (O.params.curvature_rescaling): O.p_as_x = flex.sqrt(O.curvs_work) else: O.p_as_x = flex.double(O.curvs_work.size(), 1) O.pack_parameters() O.number_of_function_evaluations = -1 O.number_of_lbfgs_iterations = -1 O.f_start, O.g_start = O.compute_functional_and_gradients() O.history = [] O.callback_after_step(minimizer=None) if (params.minimizer == "lbfgs"): O.minimizer = scitbx.lbfgs.run( target_evaluator=O, termination_params=lbfgs_termination_params, exception_handling_params=lbfgs_exception_handling_params) elif (params.minimizer == "lbfgs_raw"): O.run_lbfgs_raw() elif (params.minimizer == "lbfgsb"): O.run_lbfgsb() O.f_final, O.g_final = O.compute_functional_and_gradients() def pack_parameters(O): O.x = flex.double() O.l = flex.double() O.u = flex.double() O.nbd = flex.int() sstab = O.xray_structure.site_symmetry_table() for i_sc, sc in enumerate(O.xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() # site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): p = sc.site else: p = site_symmetry.site_constraints().independent_params( all_params=sc.site) O.x.extend(flex.double(p)) O.l.resize(O.x.size(), 0) O.u.resize(O.x.size(), 0) O.nbd.resize(O.x.size(), 0) # O.x.append(sc.u_iso) O.l.append(0) O.u.append(0) O.nbd.append(1) O.x *= O.p_as_x def unpack_parameters(O): p = O.x / O.p_as_x ix = 0 sstab = O.xray_structure.site_symmetry_table() for i_sc, sc in enumerate(O.xray_structure.scatterers()): site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): sc.site = tuple(p[ix:ix + 3]) ix += 3 else: constr = site_symmetry.site_constraints() np = constr.n_independent_params() sc.site = constr.all_params(independent_params=tuple(p[ix:ix + np])) ix += np sc.u_iso = p[ix] ix += 1 assert ix == O.x.size() def adjust_stp(O, stp, csd, large_shift_site=0.1, large_shift_u_iso_factor=0.5, min_large_shift_u_iso=b_as_u(1)): print "adjust_stp", stp assert csd.size() == O.x.size() max_stp = 100 ix = 0 uc = O.xray_structure.unit_cell() sstab = O.xray_structure.site_symmetry_table() for i_sc, sc in enumerate(O.xray_structure.scatterers()): site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 for i, ucp in enumerate(uc.parameters()[:3]): lsx = large_shift_site / ucp \ * O.p_as_x[ix+i] if (abs(stp * csd[ix + i]) > lsx): max_stp = min(max_stp, abs(lsx / csd[ix])) else: constr = site_symmetry.site_constraints() np = constr.n_independent_params() raise RuntimeError( "SPECIAL POSITION LARGE SHIFT NOT IMPLEMENTED.") ix += np u_iso = O.x[ix] / O.p_as_x[ix] lux = max(min_large_shift_u_iso, abs(u_iso) * large_shift_u_iso_factor) \ * O.p_as_x[ix] if (abs(stp * csd[ix]) > lux): max_stp = min(max_stp, abs(lux / csd[ix])) ix += 1 assert ix == O.x.size() print "max_stp:", max_stp sys.stdout.flush() if (O.params.use_max_stp): return min(max_stp, stp) return stp def compute_functional_and_gradients(O): O.number_of_function_evaluations += 1 O.unpack_parameters() try: f_calc = O.f_obs.structure_factors_from_scatterers( xray_structure=O.xray_structure, algorithm="direct", cos_sin_table=False).f_calc() except RuntimeError, e: print "RuntimeError f_calc:", e sys.stdout.flush() raise RuntimeError("f_calc") ls = xray.targets_least_squares(compute_scale_using_all_data=False, obs_type="F", obs=O.f_obs.data(), weights=O.weights, r_free_flags=None, f_calc=f_calc.data(), derivatives_depth=2, scale_factor=1) gact = O.xray_structure.grads_and_curvs_target_simple( miller_indices=O.f_obs.indices(), da_db=ls.gradients_work(), daa_dbb_dab=ls.hessians_work()) g_all = gact.grads g_active = flex.double() i_all = 0 sstab = O.xray_structure.site_symmetry_table() for i_sc, sc in enumerate(O.xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 else: np = site_symmetry.site_constraints().n_independent_params() g_active.extend(g_all[i_all:i_all + np + 1]) np += 4 # u_iso, occ, fp, fdp i_all += np assert i_all == g_all.size() assert g_active.size() == O.x.size() O.f_last = ls.target_work() O.g_last = g_active / O.p_as_x return O.f_last, O.g_last
def __get_tg(O, f_cbs, derivatives_depth=2, target=Auto, weights=Auto): if (target is Auto): target = O.params.target if (weights is Auto): weights = O.weights if (target.obs_type == "F"): obs = O.f_obs else: obs = O.i_obs if (target.type == "ls"): if (target.weighting_scheme in ["unit", "shelxl_wght_once"]): return xray.targets_least_squares( compute_scale_using_all_data=True, obs_type=target.obs_type, obs=obs.data(), weights=O.weights, r_free_flags=None, f_calc=f_cbs.data(), derivatives_depth=derivatives_depth, scale_factor=0) if (target.weighting_scheme != "shelxl_wght"): raise RuntimeError( "Unknown: target.weighting_scheme = %s" % target.weighting_scheme) assert target.obs_type == "I" if (derivatives_depth == 0): return O.calc_shelxl_wght_ls(f_cbs=f_cbs.data(), need="t") return xray.targets.shelxl_wght_ls( f_obs=O.f_obs.data(), i_obs=O.i_obs.data(), i_sig=O.i_obs.sigmas(), f_calc=f_cbs.data(), i_calc=None, wa=0.1, wb=0) elif (target.type == "cc"): return xray.targets_correlation( obs_type=target.obs_type, obs=obs.data(), weights=O.weights, r_free_flags=None, f_calc=f_cbs.data(), derivatives_depth=derivatives_depth) elif (target.type == "r1"): assert target.obs_type == "F" assert target.weighting_scheme == "unit" from cctbx.xray.targets import r1 return r1.target( f_obs=O.f_obs.data(), f_calc=f_cbs.data()) elif (target.type == "ml"): assert derivatives_depth in [0,1] if (O.epsilons is None): raise RuntimeError( "target.type=ml can only be used if bulk_solvent_correction=True") return xray.targets_maximum_likelihood_criterion( f_obs=O.f_obs.data(), r_free_flags=O.r_free_flags.data(), f_calc=f_cbs.data(), alpha=O.alpha_beta[0].data(), beta=O.alpha_beta[1].data(), scale_factor=1, epsilons=O.epsilons.data().as_double(), centric_flags=O.centric_flags.data(), compute_gradients=bool(derivatives_depth)) raise RuntimeError("Unknown target_type: %s" % target.type)
def get_curvs_work(f_obs, weights, xray_structure, lim_eps): f_calc = f_obs.structure_factors_from_scatterers( xray_structure=xray_structure, algorithm="direct", cos_sin_table=False).f_calc() ls = xray.targets_least_squares( compute_scale_using_all_data=False, obs_type="F", obs=abs(f_calc).data(), weights=weights, r_free_flags=None, f_calc=f_calc.data(), derivatives_depth=2, scale_factor=1) gact = xray_structure.grads_and_curvs_target_simple( miller_indices=f_obs.indices(), da_db=ls.gradients_work(), daa_dbb_dab=ls.hessians_work()) c_all = gact.curvs c_active_site = flex.double() c_active_u_iso = flex.double() i_all = 0 sstab = xray_structure.site_symmetry_table() for i_sc,sc in enumerate(xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 else: np = site_symmetry.site_constraints().n_independent_params() c_active_site.extend(c_all[i_all:i_all+np]) c_active_u_iso.append(c_all[i_all+np]) np += 4 # u_iso, occ, fp, fdp i_all += np assert i_all == c_all.size() # cf_site = curv_filter(curvs=c_active_site, lim_eps=lim_eps) cf_u_iso = curv_filter(curvs=c_active_u_iso, lim_eps=lim_eps) # result = flex.double() i_all = 0 sstab = xray_structure.site_symmetry_table() for i_sc,sc in enumerate(xray_structure.scatterers()): assert sc.flags.use_u_iso() assert not sc.flags.use_u_aniso() site_symmetry = sstab.get(i_sc) if (site_symmetry.is_point_group_1()): np = 3 else: np = site_symmetry.site_constraints().n_independent_params() result.extend(cf_site.apply(c_all[i_all:i_all+np])) result.extend(cf_u_iso.apply(c_all[i_all+np:i_all+np+1])) np += 4 # u_iso, occ, fp, fdp i_all += np assert i_all == c_all.size() print "curv site below limit: %d of %d" % ( cf_site.n_below_limit, cf_site.n_below_limit + cf_site.n_above_limit) print "curv u_iso below limit: %d of %d" % ( cf_u_iso.n_below_limit, cf_u_iso.n_below_limit + cf_u_iso.n_above_limit) sys.stdout.flush() return result
def __get_tg(O, f_cbs, derivatives_depth=2, target=Auto, weights=Auto): if (target is Auto): target = O.params.target if (weights is Auto): weights = O.weights if (target.obs_type == "F"): obs = O.f_obs else: obs = O.i_obs if (target.type == "ls"): if (target.weighting_scheme in ["unit", "shelxl_wght_once"]): return xray.targets_least_squares( compute_scale_using_all_data=True, obs_type=target.obs_type, obs=obs.data(), weights=O.weights, r_free_flags=None, f_calc=f_cbs.data(), derivatives_depth=derivatives_depth, scale_factor=0) if (target.weighting_scheme != "shelxl_wght"): raise RuntimeError( "Unknown: target.weighting_scheme = %s" % target.weighting_scheme) assert target.obs_type == "I" if (derivatives_depth == 0): return O.calc_shelxl_wght_ls(f_cbs=f_cbs.data(), need="t") return xray.targets.shelxl_wght_ls( f_obs=O.f_obs.data(), i_obs=O.i_obs.data(), i_sig=O.i_obs.sigmas(), f_calc=f_cbs.data(), i_calc=None, wa=0.1, wb=0) elif (target.type == "cc"): return xray.targets_correlation( obs_type=target.obs_type, obs=obs.data(), weights=O.weights, r_free_flags=None, f_calc=f_cbs.data(), derivatives_depth=derivatives_depth) elif (target.type == "r1"): assert target.obs_type == "F" assert target.weighting_scheme == "unit" from cctbx.xray.targets import r1 return r1.target( f_obs=O.f_obs.data(), f_calc=f_cbs.data()) elif (target.type == "ml"): assert derivatives_depth in [0,1] if (O.epsilons is None): raise RuntimeError( "target.type=ml can only be used if bulk_solvent_correction=True") return xray.mlf_target_and_gradients( f_obs=O.f_obs.data(), r_free_flags=O.r_free_flags.data(), f_calc=f_cbs.data(), alpha=O.alpha_beta[0].data(), beta=O.alpha_beta[1].data(), scale_factor=1, epsilons=O.epsilons.data().as_double(), centric_flags=O.centric_flags.data(), compute_gradients=bool(derivatives_depth)) raise RuntimeError("Unknown target_type: %s" % target.type)