def thin_ds1(idx, frame_params):
from libtbx.phil import parse
from simtbx.diffBragg.phil import philz
from simtbx.diffBragg.phil import hopper_phil
philz = hopper_phil + philz
phil_scope_db = parse(philz)
from LS49.adse13_187.report_versions import ds1_params_v4 as ds1_params
user_phil = parse(ds1_params)
working_phil = phil_scope_db.fetch(sources=[user_phil])
params = working_phil.extract()
from simtbx.diffBragg import hopper_utils
BERNINA = os.environ.get("BERNINA")
try:
exp, ref, data_modeler, x = hopper_utils.refine(
exp=os.path.join(BERNINA, "split_cs", "split_%04d.expt"%idx),
ref = os.path.join(BERNINA, "split2b" , "split_%04d.refl"%idx),
params=params, return_modeler=True)
(scale, rotX, rotY, rotZ, Na, Nb, Nc,
diff_gam_a, diff_gam_b, diff_gam_c, diff_sig_a, diff_sig_b, diff_sig_c,
a,b,c,al,be,ga, detz) = hopper_utils.get_param_from_x(x,data_modeler.SIM)
from dxtbx.model.experiment_list import ExperimentList
L = ExperimentList()
L.append(exp)
L.as_file("ds1_%04d.expt"%idx)
print("DS1 parameters index %d Na Nb Nc %.3f %.3f %.3f"%(idx,Na,Nb,Nc))
from LS49.adse13_187.adse13_221.basic_runner import run as basic_run
anaparams = mcmc_runner_parse_input()
anaparams.trusted_mask=os.path.join(BERNINA, "trusted_Py3.mask")
anaparams.cryst="ds1_%04d.expt"%idx
anaparams.expt = os.path.join(BERNINA, "split_cs", "split_%04d.expt"%idx)
anaparams.refl=os.path.join(BERNINA, "split2b" , "split_%04d.refl"%idx)
anaparams.output.enable=False
anaparams.output.label="mcmc3"
anaparams.output.index=idx
anaparams.model.mosaic_spread.value=0.001
anaparams.model.Nabc.value=(Na,Nb,Nc)
anaparams.model.Nabc.hyperparameter=0.8
anaparams.model.rot.refine=True
anaparams.model.cell.covariance=os.path.join(
BERNINA,"..", "covariance_cytochrome_form.pickle")
anaparams.simplex.cycles=200
anaparams.mcmc.cycles=2000
anaparams.model.plot=False
print(anaparams.trusted_mask)
print(anaparams.cryst)
print(anaparams.expt)
print(anaparams.refl)
print(anaparams.model.cell.covariance)
basic_run(anaparams)
except Exception as e:
print("CATCH EXCEPTION",e)
traceback.print_exc()
def save_to_pandas(x, SIM, orig_exp_name, params, expt, rank_exp_idx, stg1_refls, stg1_img_path):
LOGGER = logging.getLogger("refine")
rank_exper_outdir = make_rank_outdir(params.outdir, "expers")
rank_pandas_outdir =make_rank_outdir(params.outdir, "pandas")
#if SIM.num_xtals > 1:
# raise NotImplemented("cant save pandas for multiple crystals yet")
scale, rotX, rotY, rotZ, Na, Nb, Nc,diff_gam_a, diff_gam_b, diff_gam_c, diff_sig_a, diff_sig_b, diff_sig_c, a,b,c,al,be,ga,detz_shift = hopper_utils.get_param_from_x(x, SIM)
if params.isotropic.diffuse_gamma:
diff_gam_b = diff_gam_c = diff_gam_a
if params.isotropic.diffuse_sigma:
diff_sig_b = diff_sig_c = diff_sig_a
shift = np.nan
#if SIM.shift_param is not None:
# shift = SIM.shift_param.get_val(x[-1])
xtal_scales = [scale]
eta_a, eta_b, eta_c = hopper_utils.get_mosaicity_from_x(x, SIM)
a_init, b_init, c_init, al_init, be_init, ga_init = SIM.crystal.dxtbx_crystal.get_unit_cell().parameters()
xax = col((-1, 0, 0))
yax = col((0, -1, 0))
zax = col((0, 0, -1))
## update parameters:
RX = xax.axis_and_angle_as_r3_rotation_matrix(rotX, deg=False)
RY = yax.axis_and_angle_as_r3_rotation_matrix(rotY, deg=False)
RZ = zax.axis_and_angle_as_r3_rotation_matrix(rotZ, deg=False)
M = RX * RY * RZ
U = M * sqr(SIM.crystal.dxtbx_crystal.get_U())
SIM.crystal.dxtbx_crystal.set_U(U)
ucparam = a, b, c, al, be, ga
ucman = utils.manager_from_params(ucparam)
SIM.crystal.dxtbx_crystal.set_B(ucman.B_recipspace)
Amats = [SIM.crystal.dxtbx_crystal.get_A()]
ncells_def_vals = [(0, 0, 0)]
ncells_vals = [(Na, Nb, Nc)]
eta = [0]
lam0 = [-1]
lam1 = [-1]
df = pandas.DataFrame({
# "panX": list(panX), "panY": list(panY), "panZ": list(panZ),
# "panO": list(panO), "panF": list(panF), "panS": list(panS),
"spot_scales": xtal_scales, "Amats": Amats, "ncells": ncells_vals,
"eta_abc": [(eta_a, eta_b, eta_c)],
"detz_shift_mm": [detz_shift * 1e3],
"ncells_def": ncells_def_vals,
"diffuse_gamma": [(diff_gam_a, diff_gam_b, diff_gam_c)],
"diffuse_sigma": [(diff_sig_a, diff_sig_b, diff_sig_c)],
"fp_fdp_shift": [shift],
"use_diffuse_models": [params.use_diffuse_models],
"gamma_miller_units": [params.gamma_miller_units],
# "bgplanes": bgplanes, "image_corr": image_corr,
# "init_image_corr": init_img_corr,
# "fcell_xstart": fcell_xstart,
# "ucell_xstart": ucell_xstart,
# "init_misorient": init_misori, "final_misorient": final_misori,
# "bg_coef": bg_coef,
"eta": eta,
"rotX": rotX,
"rotY": rotY,
"rotZ": rotZ,
"a": a, "b": b, "c": c, "al": al, "be": be, "ga": ga,
"a_init": a_init, "b_init": b_init, "c_init": c_init, "al_init": al_init,
"lam0": lam0, "lam1": lam1,
"be_init": be_init, "ga_init": ga_init})
# "scale_xpos": scale_xpos,
# "ncells_xpos": ncells_xstart,
# "bgplanes_xpos": bgplane_xpos})
basename = os.path.splitext(os.path.basename(orig_exp_name))[0]
opt_exp_path = os.path.join(rank_exper_outdir, "%s_%s_%d.expt" % (params.tag, basename, rank_exp_idx))
pandas_path = os.path.join(rank_pandas_outdir, "%s_%s_%d.pkl" % (params.tag, basename, rank_exp_idx))
expt.crystal = SIM.crystal.dxtbx_crystal
# expt.detector = refiner.get_optimized_detector()
new_exp_list = ExperimentList()
new_exp_list.append(expt)
new_exp_list.as_file(opt_exp_path)
LOGGER.debug("saved opt_exp %s with wavelength %f" % (opt_exp_path, expt.beam.get_wavelength()))
spec_file = None
if params.simulator.spectrum.filename is not None:
spec_file = os.path.abspath(params.simulator.spectrum.filename)
df["spectrum_filename"] = spec_file
df["spectrum_stride"] = params.simulator.spectrum.stride
df["total_flux"] = SIM.D.flux # params.simulator.total_flux
df["beamsize_mm"] = SIM.beam.size_mm
df["exp_name"] = os.path.abspath(orig_exp_name)
df["opt_exp_name"] = os.path.abspath(opt_exp_path)
df["spectrum_from_imageset"] = params.spectrum_from_imageset
df["oversample"] = params.simulator.oversample
if params.opt_det is not None:
df["opt_det"] = params.opt_det
df["stage1_refls"] = stg1_refls
df["stage1_output_img"] = stg1_img_path
df.to_pickle(pandas_path)
return df
if "eta" in args.perturb:
P.ftol = 1e-8
#P.method="Nelder-Mead"
#P.fix.G = True
#P.fix.Nabc =True
#P.fix.detz_shift=True
import logging
import sys
h = logging.StreamHandler(sys.stdout)
logging.basicConfig(level=logging.DEBUG, handlers=[h])
from simtbx.diffBragg import hopper_utils
Eopt, _, Mod, x = hopper_utils.refine(E, refls, P, return_modeler=True)
G, rotX, rotY, rotZ, Na, Nb, Nc, _, _, _, _, _, _, a, b, c, al, be, ga, detz_shift = hopper_utils.get_param_from_x(
x, Mod.SIM)
eta_abc_opt = hopper_utils.get_mosaicity_from_x(x, Mod.SIM)
print("Na, Nb, Nc= %f %f %f" % (Na, Nb, Nc))
print("eta_abc optimized:", eta_abc_opt)
# check crystal
Copt = Eopt.crystal
misset, misset_init = utils.compare_with_ground_truth(
*C.get_real_space_vectors(),
dxcryst_models=[Copt, E.crystal],
symbol=symbol)
print(misset_init, "init misset with ground truth")
print(misset, "misset with ground truth")
if "detz_shift" in args.perturb:
assert misset < 0.007, misset
def tst_ds1(idx, frame_params):
print("in tst_diffbragg_stage_1")
from libtbx.phil import parse
from simtbx.diffBragg.phil import philz
from simtbx.diffBragg.phil import hopper_phil
philz = hopper_phil + philz
phil_scope_db = parse(philz)
user_phil = parse(ds1_params)
working_phil = phil_scope_db.fetch(sources=[user_phil])
params = working_phil.extract()
#import logging
#logging.disable(level=logging.CRITICAL)
#logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
from simtbx.diffBragg import hopper_utils
BERNINA = os.environ.get("BERNINA")
#params.fix.detz_shift = True
print ("hopper",flush=True)
try:
exp, ref, data_modeler, x = hopper_utils.refine(
exp=os.path.join(BERNINA, "split_cs", "split_%04d.expt"%idx),
ref = os.path.join(BERNINA, "split2b" , "split_%04d.refl"%idx),
params=params, return_modeler=True)
print("1",flush=True)
scale, rotX, rotY, rotZ, Na, Nb, Nc,diff_gam_a, diff_gam_b, diff_gam_c, diff_sig_a, diff_sig_b, diff_sig_c, a,b,c,al,be,ga, detz = hopper_utils.get_param_from_x(x,data_modeler.SIM)
print("The parameters are\n",scale, rotX, rotY, rotZ, Na, Nb, Nc,diff_gam_a, diff_gam_b, diff_gam_c, diff_sig_a, diff_sig_b, diff_sig_c, a,b,c,al,be,ga, detz)
except Exception as e:
print("CATCH EXCEPTION",e)
traceback.print_tb(e.__traceback__)
def save_to_pandas(x, SIM, orig_exp_name, params, expt, rank_exp_idx,
stg1_refls, stg1_img_path):
LOGGER = logging.getLogger("refine")
rank_exper_outdir = make_rank_outdir(params.outdir, "expers")
rank_pandas_outdir = make_rank_outdir(params.outdir, "pandas")
scale, rotX, rotY, rotZ, Na, Nb, Nc, diff_gam_a, diff_gam_b, diff_gam_c, diff_sig_a, diff_sig_b, diff_sig_c, a, b, c, al, be, ga, detz_shift = hopper_utils.get_param_from_x(
x, SIM)
scale_p = SIM.P["G_xtal0"]
scale_init = scale_p.init
Nabc_init = []
for i in [0, 1, 2]:
p = SIM.P["Nabc%d" % i]
Nabc_init.append(p.init)
Nabc_init = tuple(Nabc_init)
if params.isotropic.diffuse_gamma:
diff_gam_b = diff_gam_c = diff_gam_a
if params.isotropic.diffuse_sigma:
diff_sig_b = diff_sig_c = diff_sig_a
eta_a, eta_b, eta_c = hopper_utils.get_mosaicity_from_x(x, SIM)
a_init, b_init, c_init, al_init, be_init, ga_init = SIM.crystal.dxtbx_crystal.get_unit_cell(
).parameters()
xax = col((-1, 0, 0))
yax = col((0, -1, 0))
zax = col((0, 0, -1))
## update parameters:
RX = xax.axis_and_angle_as_r3_rotation_matrix(rotX, deg=False)
RY = yax.axis_and_angle_as_r3_rotation_matrix(rotY, deg=False)
RZ = zax.axis_and_angle_as_r3_rotation_matrix(rotZ, deg=False)
M = RX * RY * RZ
U = M * sqr(SIM.crystal.dxtbx_crystal.get_U())
SIM.crystal.dxtbx_crystal.set_U(U)
ucparam = a, b, c, al, be, ga
ucman = utils.manager_from_params(ucparam)
SIM.crystal.dxtbx_crystal.set_B(ucman.B_recipspace)
Amat = SIM.crystal.dxtbx_crystal.get_A()
eta = [0]
lam_coefs = [0], [1]
if hasattr(SIM, "P"):
names = "lambda_offset", "lambda_scale"
if names[0] in SIM.P and names[1] in SIM.P:
lam_coefs = []
for name in names:
if name in SIM.P:
p = SIM.P[name]
val = p.get_val(x[p.xpos])
lam_coefs.append([val])
lam_coefs = tuple(lam_coefs)
basename = os.path.splitext(os.path.basename(orig_exp_name))[0]
opt_exp_path = os.path.join(
rank_exper_outdir,
"%s_%s_%d.expt" % (params.tag, basename, rank_exp_idx))
pandas_path = os.path.join(
rank_pandas_outdir,
"%s_%s_%d.pkl" % (params.tag, basename, rank_exp_idx))
expt.crystal = SIM.crystal.dxtbx_crystal
# expt.detector = refiner.get_optimized_detector()
new_exp_list = ExperimentList()
new_exp_list.append(expt)
new_exp_list.as_file(opt_exp_path)
LOGGER.debug("saved opt_exp %s with wavelength %f" %
(opt_exp_path, expt.beam.get_wavelength()))
_, flux_vals = zip(*SIM.beam.spectrum)
df = single_expt_pandas(xtal_scale=scale,
Amat=Amat,
ncells_abc=(Na, Nb, Nc),
ncells_def=(0, 0, 0),
eta_abc=(eta_a, eta_b, eta_c),
diff_gamma=(diff_gam_a, diff_gam_b, diff_gam_c),
diff_sigma=(diff_sig_a, diff_sig_b, diff_sig_c),
detz_shift=detz_shift,
use_diffuse=params.use_diffuse_models,
gamma_miller_units=params.gamma_miller_units,
eta=eta,
rotXYZ=(rotX, rotY, rotZ),
ucell_p=(a, b, c, al, be, ga),
ucell_p_init=(a_init, b_init, c_init, al_init,
be_init, ga_init),
lam0_lam1=lam_coefs,
spec_file=params.simulator.spectrum.filename,
spec_stride=params.simulator.spectrum.stride,
flux=sum(flux_vals),
beamsize_mm=SIM.beam.size_mm,
orig_exp_name=orig_exp_name,
opt_exp_name=opt_exp_path,
spec_from_imageset=params.spectrum_from_imageset,
oversample=params.simulator.oversample,
opt_det=params.opt_det,
stg1_refls=stg1_refls,
stg1_img_path=stg1_img_path,
ncells_init=Nabc_init,
spot_scales_init=scale_init)
df.to_pickle(pandas_path)
return df