def run_one(space_group_info, n_elements=9, volume_per_atom=1000, d_min=2.0, anomalous_flag=0, verbose=0): if (random.random() < 0.5): random_f_prime_scale = 0.6 else: random_f_prime_scale = 0 structure_ideal = random_structure.xray_structure( space_group_info, elements=( ("O", "N", "C") * (n_elements) )[:n_elements], #(("O","N","C")*(n_elements/3+1))[:n_elements], volume_per_atom=volume_per_atom, min_distance=5, general_positions_only=True, random_f_prime_d_min=d_min - 1, random_f_prime_scale=random_f_prime_scale, random_f_double_prime=anomalous_flag, use_u_aniso=True, use_u_iso=False, random_u_iso=True, random_u_cart_scale=.3, random_occupancy=True) random_structure.random_modify_adp_and_adp_flags( scatterers=structure_ideal.scatterers(), random_u_iso_scale=0.3, random_u_iso_min=0.0) if (random.random() < 0.5): assign_custom_gaussians(structure_ideal, negative_a=random.random() < 0.5) if (0 or verbose): structure_ideal.show_summary().show_scatterers() f_obs = abs( structure_ideal.structure_factors(d_min=d_min, anomalous_flag=anomalous_flag, algorithm="direct").f_calc()) if (1): site(structure_ideal, d_min, f_obs, verbose=verbose) if (1): u_iso(structure_ideal, d_min, f_obs, tan_u_iso=False, verbose=verbose) u_iso(structure_ideal, d_min, f_obs, tan_u_iso=True, verbose=verbose) u_star(structure_ideal, d_min, f_obs, verbose=verbose) if (1): occupancy(structure_ideal, d_min, f_obs, verbose=verbose) if (1): fp(structure_ideal, d_min, f_obs, verbose=verbose) if (1 and anomalous_flag): fdp(structure_ideal, d_min, f_obs, verbose=verbose) if (1): exercise_gradient_manager(structure_ideal, f_obs, anomalous_flag) if (1): exercise_packed(structure_ideal, f_obs, anomalous_flag)
def run_one(space_group_info, n_elements= 9, volume_per_atom=1000, d_min = 2.0, anomalous_flag=0, verbose=0): if (random.random() < 0.5): random_f_prime_scale=0.6 else: random_f_prime_scale=0 structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements],#(("O","N","C")*(n_elements/3+1))[:n_elements], volume_per_atom=volume_per_atom, min_distance=5, general_positions_only=True, random_f_prime_d_min=d_min-1, random_f_prime_scale=random_f_prime_scale, random_f_double_prime=anomalous_flag, use_u_aniso = True, use_u_iso = False, random_u_iso=True, random_u_cart_scale=.3, random_occupancy=True) random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0) if (random.random() < 0.5): assign_custom_gaussians(structure_ideal, negative_a=random.random()<0.5) if (0 or verbose): structure_ideal.show_summary().show_scatterers() f_obs = abs(structure_ideal.structure_factors( d_min=d_min, anomalous_flag=anomalous_flag, algorithm="direct").f_calc()) if (1): site(structure_ideal, d_min, f_obs, verbose=verbose) if (1): u_iso(structure_ideal, d_min, f_obs, tan_u_iso=False, verbose=verbose) u_iso(structure_ideal, d_min, f_obs, tan_u_iso=True, verbose=verbose) u_star(structure_ideal, d_min, f_obs, verbose=verbose) if (1): occupancy(structure_ideal, d_min, f_obs, verbose=verbose) if (1): fp(structure_ideal, d_min, f_obs, verbose=verbose) if (1 and anomalous_flag): fdp(structure_ideal, d_min, f_obs, verbose=verbose) if (1): exercise_gradient_manager(structure_ideal, f_obs, anomalous_flag) if (1): exercise_packed(structure_ideal, f_obs, anomalous_flag)
def exercise(target_functor, data_type, space_group_info, anomalous_flag, gradient_flags, occupancy_penalty, n_elements=9, d_min=None, shake_sigma=0.1, verbose=0,tan_u_iso=False, param = 0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements],#("Se",)*n_elements, volume_per_atom=200, random_u_iso=True, random_u_cart_scale=.3, random_occupancy=True, use_u_aniso = True) random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0) xray.set_scatterer_grad_flags(scatterers = structure_ideal.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp, tan_u_iso = tan_u_iso, param = param) if(0): print for sc in structure_ideal.scatterers(): print sc.flags.use_u_iso(),sc.flags.grad_u_iso(),sc.flags.use_u_aniso(),\ sc.flags.grad_u_aniso(),sc.u_iso, sc.u_star,sc.flags.tan_u_iso(),\ sc.flags.param, sc.occupancy rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct", cos_sin_table=True).f_calc() if data_type == "F": y_obs = abs(rnd_f_calc) elif data_type == "F^2": y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() else: raise "Error: invalid data type: %s" % data_type if (0 or verbose): print "structure_ideal:" structure_ideal.show_summary().show_scatterers() print "n_special_positions:", \ structure_ideal.special_position_indices().size() print structure_ideal_cp = structure_ideal.deep_copy_scatterers() structure_shake = structure_ideal if (gradient_flags.site): structure_shake = structure_shake.random_modify_parameters( "site", shake_sigma) if (gradient_flags.occupancy): structure_shake = structure_shake.random_modify_parameters( "occupancy", shake_sigma) if (occupancy_penalty is not None): structure_shake.scatterers()[-1].occupancy = 0 if (gradient_flags.u_aniso): shift_u_aniso(structure_shake, 0.001) if (gradient_flags.u_iso): shift_u_iso(structure_shake, 0.1) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) if (0 or verbose): print "structure_shake:" structure_shake.show_summary().show_scatterers() print for i_trial in xrange(10): try: minimizer = xray.minimization.lbfgs( target_functor=target_functor(y_obs), xray_structure=structure_shake, occupancy_penalty=occupancy_penalty, structure_factor_algorithm="direct") except RuntimeError, e: if (str(e).find("debye_waller_factor_exp: arg_limit exceeded") < 0): raise else: break
def exercise(target_functor, data_type, parameter_name, space_group_info, anomalous_flag, cartesian_flag, n_elements=9, d_min=2.5, shake_sigma=0.25, test_hard=True, verbose=0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return if (parameter_name != "site" and cartesian_flag == True): return if (parameter_name == "fdp" and not anomalous_flag): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O", "N", "C") * (n_elements))[:n_elements], volume_per_atom=100, random_f_prime_d_min=d_min, random_f_double_prime=anomalous_flag, use_u_aniso=True, use_u_iso=False, random_u_cart_scale=0.3, random_u_iso=False, random_occupancy=True) if (parameter_name in ["u_star", "u_iso"]): shake_sigma = shake_sigma / 2. random_structure.random_modify_adp_and_adp_flags( scatterers=structure_ideal.scatterers(), random_u_iso_scale=0.3, random_u_iso_min=0.0, parameter_name=parameter_name) rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() if data_type == 'F': y_obs = abs(rnd_f_calc) elif data_type == 'F^2': y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() structure_shake = structure_ideal.random_modify_parameters( parameter_name, shake_sigma, vary_z_only=False) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) target_ftor = target_functor(y_obs) for structure in (structure_ideal, structure_shake)[:]: #SWITCH f_calc = y_obs.structure_factors_from_scatterers( xray_structure=structure, algorithm="direct").f_calc() target_result = target_ftor(f_calc, compute_derivatives=True) if (structure == structure_ideal): assert abs(target_result.target()) < 1.e-5 gradient_flags = xray.structure_factors.gradient_flags( site=(parameter_name == "site" or random.choice((False, True))), u_iso=(parameter_name == "u_iso" or random.choice((False, True))), u_aniso=(parameter_name == "u_star" or random.choice((False, True))), occupancy=(parameter_name == "occupancy" or random.choice( (False, True))), fp=(parameter_name == "fp" or random.choice((False, True))), fdp=(parameter_name == "fdp" or (anomalous_flag and random.choice( (False, True))))) xray.set_scatterer_grad_flags(scatterers=structure.scatterers(), site=gradient_flags.site, u_iso=gradient_flags.u_iso, u_aniso=gradient_flags.u_aniso, occupancy=gradient_flags.occupancy, fp=gradient_flags.fp, fdp=gradient_flags.fdp) grad_flags_counts = xray.scatterer_grad_flags_counts( structure.scatterers()) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params=None, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) if (parameter_name == "site"): d_analytical = sf.d_target_d_site_frac() if (cartesian_flag ): # average d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site( d_analytical) if (cartesian_flag): d_analytical = sf.d_target_d_site_cart() d_numerical = finite_differences_site(cartesian_flag, target_ftor, structure) if (not cartesian_flag ): # aver. d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site( d_numerical) elif (parameter_name == "u_star" and grad_flags_counts.use_u_aniso > 0): d_analytical = sf.d_target_d_u_star() d_numerical = finite_differences_u_star(target_ftor, structure) else: if (parameter_name == "occupancy"): d_analytical = sf.d_target_d_occupancy() elif (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): d_analytical = sf.d_target_d_u_iso() elif (parameter_name == "fp"): d_analytical = sf.d_target_d_fp() elif (parameter_name == "fdp"): d_analytical = sf.d_target_d_fdp() else: raise RuntimeError d_numerical = finite_differences_scalar(parameter_name, target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose) if (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): u_iso_refinable_params = flex.double() for scatterer in structure.scatterers(): scatterer.flags.set_tan_u_iso(True) scatterer.flags.set_grad_u_iso(gradient_flags.u_iso) param = random.randint(90, 120) scatterer.flags.param = param value = math.tan(scatterer.u_iso * math.pi / adptbx.b_as_u(param) - math.pi / 2) u_iso_refinable_params.append(value) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params=u_iso_refinable_params, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) d_analytical = sf.d_target_d_u_iso() d_numerical = finite_differences_scalar("tan_u_iso", target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose)
def exercise(target_functor, data_type, space_group_info, anomalous_flag, gradient_flags, occupancy_penalty, n_elements=9, d_min=None, shake_sigma=0.1, verbose=0,tan_u_iso=False, param = 0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements],#("Se",)*n_elements, volume_per_atom=200, random_u_iso=True, random_u_cart_scale=.3, random_occupancy=True, use_u_aniso = True) random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0) xray.set_scatterer_grad_flags(scatterers = structure_ideal.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp, tan_u_iso = tan_u_iso, param = param) if(0): print() for sc in structure_ideal.scatterers(): print(sc.flags.use_u_iso(),sc.flags.grad_u_iso(),sc.flags.use_u_aniso(),\ sc.flags.grad_u_aniso(),sc.u_iso, sc.u_star,sc.flags.tan_u_iso(),\ sc.flags.param, sc.occupancy) rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct", cos_sin_table=True).f_calc() if data_type == "F": y_obs = abs(rnd_f_calc) elif data_type == "F^2": y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() else: raise "Error: invalid data type: %s" % data_type if (0 or verbose): print("structure_ideal:") structure_ideal.show_summary().show_scatterers() print("n_special_positions:", \ structure_ideal.special_position_indices().size()) print() structure_ideal_cp = structure_ideal.deep_copy_scatterers() structure_shake = structure_ideal if (gradient_flags.site): structure_shake = structure_shake.random_modify_parameters( "site", shake_sigma) if (gradient_flags.occupancy): structure_shake = structure_shake.random_modify_parameters( "occupancy", shake_sigma) if (occupancy_penalty is not None): structure_shake.scatterers()[-1].occupancy = 0 if (gradient_flags.u_aniso): shift_u_aniso(structure_shake, 0.001) if (gradient_flags.u_iso): shift_u_iso(structure_shake, 0.1) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) if (0 or verbose): print("structure_shake:") structure_shake.show_summary().show_scatterers() print() for i_trial in range(10): try: minimizer = xray.minimization.lbfgs( target_functor=target_functor(y_obs), xray_structure=structure_shake, occupancy_penalty=occupancy_penalty, structure_factor_algorithm="direct") except RuntimeError as e: if (str(e).find("debye_waller_factor_exp: arg_limit exceeded") < 0): raise else: break else: raise RuntimeError("Too many xray.minimization.lbfgs failures.") if (0 or verbose): print("first:", minimizer.first_target_value) print("final:", minimizer.final_target_value) print() assert minimizer.final_target_value < minimizer.first_target_value if (0 or verbose): print("minimized structure_shake:") structure_shake.show_summary().show_scatterers() print() f_final = y_obs.structure_factors_from_scatterers( xray_structure=structure_shake, algorithm="direct", cos_sin_table=True).f_calc() if data_type == 'F': f_final = abs(f_final) elif data_type == 'F^2': f_final = f_final.norm() c = flex.linear_correlation(y_obs.data(), f_final.data()) assert c.is_well_defined() if (0 or verbose): label = gradient_flags.string_of_true() if (anomalous_flag): label += ",anomalous" print("correlation: %10.8f" % c.coefficient(), label) print() c_coefficient = c.coefficient() if(c_coefficient <= 0.999): print(c_coefficient) if data_type == 'F': assert c_coefficient > 0.999 elif data_type == 'F^2': assert c_coefficient > 0.9
def exercise(target_functor, data_type, parameter_name, space_group_info, anomalous_flag, cartesian_flag, n_elements=9, d_min=2.5, shake_sigma=0.25, test_hard=True, verbose=0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return if (parameter_name != "site" and cartesian_flag == True): return if (parameter_name == "fdp" and not anomalous_flag): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements], volume_per_atom=100, random_f_prime_d_min=d_min, random_f_double_prime=anomalous_flag, use_u_aniso = True, use_u_iso = False, random_u_cart_scale = 0.3, random_u_iso = False, random_occupancy=True) if(parameter_name in ["u_star", "u_iso"]): shake_sigma = shake_sigma / 2. random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0, parameter_name = parameter_name) rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() if data_type == 'F': y_obs = abs(rnd_f_calc) elif data_type == 'F^2': y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() structure_shake = structure_ideal.random_modify_parameters( parameter_name, shake_sigma, vary_z_only=False) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) target_ftor = target_functor(y_obs) for structure in (structure_ideal, structure_shake)[:]: #SWITCH f_calc = y_obs.structure_factors_from_scatterers( xray_structure=structure, algorithm="direct").f_calc() target_result = target_ftor(f_calc, compute_derivatives=True) if (structure == structure_ideal): assert abs(target_result.target()) < 1.e-5 gradient_flags=xray.structure_factors.gradient_flags( site=(parameter_name=="site" or random.choice((False,True))), u_iso=(parameter_name=="u_iso" or random.choice((False,True))), u_aniso=(parameter_name=="u_star" or random.choice((False,True))), occupancy=(parameter_name=="occupancy" or random.choice((False,True))), fp=(parameter_name=="fp" or random.choice((False,True))), fdp=(parameter_name=="fdp" or (anomalous_flag and random.choice((False,True))))) xray.set_scatterer_grad_flags(scatterers = structure.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) grad_flags_counts = xray.scatterer_grad_flags_counts(structure.scatterers()) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params=None, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) if (parameter_name == "site"): d_analytical = sf.d_target_d_site_frac() if (cartesian_flag): # average d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site(d_analytical) if (cartesian_flag): d_analytical = sf.d_target_d_site_cart() d_numerical = finite_differences_site( cartesian_flag, target_ftor, structure) if (not cartesian_flag): # aver. d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site(d_numerical) elif (parameter_name == "u_star" and grad_flags_counts.use_u_aniso > 0): d_analytical = sf.d_target_d_u_star() d_numerical = finite_differences_u_star(target_ftor, structure) else: if (parameter_name == "occupancy"): d_analytical = sf.d_target_d_occupancy() elif (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): d_analytical = sf.d_target_d_u_iso() elif (parameter_name == "fp"): d_analytical = sf.d_target_d_fp() elif (parameter_name == "fdp"): d_analytical = sf.d_target_d_fdp() else: raise RuntimeError d_numerical = finite_differences_scalar( parameter_name, target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose) if (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): u_iso_refinable_params = flex.double() for scatterer in structure.scatterers(): scatterer.flags.set_tan_u_iso(True) scatterer.flags.set_grad_u_iso(gradient_flags.u_iso) param = random.randint(90,120) scatterer.flags.param= param value = math.tan(scatterer.u_iso*math.pi/adptbx.b_as_u(param)-math.pi/2) u_iso_refinable_params.append(value) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params = u_iso_refinable_params, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) d_analytical = sf.d_target_d_u_iso() d_numerical = finite_differences_scalar("tan_u_iso",target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose)