def unload_compress_masks(fname):
"""
unpacks the fname and stores the masks in dials format
:param fname: a compress_mask files created using
compress_masks
"""
with h5py.File(fname, "r") as h5:
for name in h5.keys():
if os.path.exists(name):
print("Path %s exists!, skipping" % name)
continue
data = h5[name][()]
if data.dtype != bool:
continue
data = tuple([flex.bool(m) for m in data])
utils.save_flex(data, name)
print 'adding in anom scattering for yb'
for i, s in enumerate(sc):
if i in yb_pos:
fp, fdp = scattering_factors.elem_fp_fdp_at_eV(s.element_symbol(),
args.e,
how='henke')
s.fp = fp
s.fdp = fdp
#fp, fdp = scattering_factors.Yb_fp_fdp_at_eV(energy_eV, 'henke')
#sc[yb_pos[0]].fp = fp
#sc[yb_pos[0]].fdp = fdp
#sc[yb_pos[1]].fp = fp
#sc[yb_pos[1]].fdp = fdp
#xr2_sel = flex.bool(len(sc), True)
#for pos in yb_pos:
# xr2_sel[pos] = False
#xr2 = xr.select(xr2_sel)
Ftot = xr.structure_factors(d_min=d_min,
algorithm='direct',
anomalous_flag=True).f_calc()
Atot = Ftot.data().as_numpy_array()
#print "Ftotal and Fprot + Fheavy should be equalish!"
#assert (np.allclose(Atot, Aa+Ap))
out = {"Aprotein": Fp, "Aheavy": Fa, "Atotal": Ftot}
utils.save_flex(out, args.o)
def msi(jid):
"""
msi stands for mad --> spot --> index
Here we load a data image that has crystal diffraction
then we spot peaks and index the crystal image
"""
#time.sleep(jid)
import sys
import numpy as np
import glob
import os
from copy import deepcopy
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dials.command_line.stills_process import phil_scope\
as indexer_phil_scope
from dials.algorithms.indexing.lattice_search import basis_vector_search_phil_scope
from dials.array_family import flex
import dxtbx
from dxtbx.datablock import DataBlockFactory
from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList, Experiment
from libtbx.utils import Sorry as Sorry
from libtbx.phil import parse
from cxi_xdr_xes.command_line.two_color_process import two_color_phil_scope
from cxi_xdr_xes.two_color import two_color_indexer
indexer_two_color = two_color_indexer.indexer_two_color
from cctbx import crystal
from cxid9114 import utils
from cxid9114 import parameters
indexer_phil_scope.adopt_scope(two_color_phil_scope)
indexer_phil_scope.adopt_scope(basis_vector_search_phil_scope)
mad_index_params = indexer_phil_scope.extract()
fnames = glob.glob(glob_str)
out_dir = os.path.join( args.out_dir , "job%d" % jid)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
refls_dir = "refls_%s" % tag
Els_dir = "Els_%s" % tag
dumps_dir = "dumps_%s" % tag
for o in [refls_dir, Els_dir, dumps_dir]:
sub_o = os.path.join(out_dir, o)
if not os.path.exists(sub_o):
os.makedirs(sub_o)
# track shots that indexed, or shots that
# had too few spots to index, so can change parameters and try again
def load_tracker_f(fname):
data = []
if os.path.exists(fname):
data = np.loadtxt(fname, str)
if data.size and not data.shape:
data = list(set(data[None].astype(int)))
else:
data = list(set(data.astype(int)))
return data
skip_weak = False
skip_failed = True #False
skip_indexed = False
weak_shots_f = os.path.join(out_dir, "weak_shots.txt")
failed_idx_f = os.path.join(out_dir, "failed_shots.txt")
indexed_f = os.path.join(out_dir, "indexed_shots.txt")
weak_shots = load_tracker_f(weak_shots_f)
failed_shots = load_tracker_f(failed_idx_f)
indexed_shots = load_tracker_f(indexed_f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# ------ spotting parameters
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par.spotfinder.threshold.dispersion.global_threshold = 40
spot_par.spotfinder.threshold.dispersion.gain = 28
spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 1
spot_par.spotfinder.threshold.dispersion.sigma_background = 6
spot_par.spotfinder.filter.min_spot_size = 3
spot_par.spotfinder.force_2d = True
# ------ indexing parameters
KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P1")
#KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P43212")
mad_index_params.refinement.parameterisation.beam.fix = "all"
mad_index_params.refinement.parameterisation.detector.fix = "all"
#mad_index_params.refinement.verbosity = 3
#mad_index_params.refinement.reflections.outlier.algorithm = "null"
mad_index_params.indexing.stills.refine_all_candidates = do_stills_refine
mad_index_params.indexing.optimise_initial_basis_vectors = do_basis_refine
#mad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False
mad_index_params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info()
mad_index_params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell()
mad_index_params.indexing.refinement_protocol.d_min_start = None
mad_index_params.indexing.debug = True
mad_index_params.indexing.real_space_grid_search.characteristic_grid = 0.02
mad_index_params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
mad_index_params.indexing.known_symmetry.relative_length_tolerance = 0.3
mad_index_params.indexing.stills.rmsd_min_px = 20000
mad_index_params.indexing.refinement_protocol.n_macro_cycles = 1
mad_index_params.indexing.multiple_lattice_search.max_lattices = 1
mad_index_params.indexing.basis_vector_combinations.max_refine = 10000000000
#mad_index_params.indexing.basis_vector_combinations.max_combinations = 150
#mad_index_params.indexing.stills.candidate_outlier_rejection = False
mad_index_params.indexing.refinement_protocol.mode = None #$"repredict_only"
mad_index_params.indexing.two_color.high_energy = parameters.ENERGY_HIGH
mad_index_params.indexing.two_color.low_energy = parameters.ENERGY_LOW
mad_index_params.indexing.two_color.avg_energy = parameters.ENERGY_LOW * .5 + parameters.ENERGY_HIGH * .5
#mad_index_params.indexing.two_color.spiral_method = (1., 100000) # 1000000)
mad_index_params.indexing.two_color.n_unique_v = 22
#mad_index_params.indexing.two_color.block_size = 25
#mad_index_params.indexing.two_color.filter_by_mag = (10,3)
# ------
N = len(fnames)
idx_split = np.array_split(np.arange(N), n_jobs)
n_idx = 0 # number indexed
for idx in idx_split[jid]:
if args.Nmax is not None:
if idx == args.Nmax:
sys.exit()
img_f = fnames[idx]
loader = dxtbx.load(img_f)
iset = loader.get_imageset(filenames=[loader.get_image_file()] )
DET = loader.get_detector()
BEAM = loader.get_beam()
El = ExperimentListFactory.from_imageset_and_crystal(iset, crystal=None)
E = El[0]
from cxid9114.geom.multi_panel import CSPAD_refined, CSPAD
El[0].detector = CSPAD_refined
if idx in weak_shots and skip_weak:
print("Skipping weak shots %d" % idx)
continue
if idx in failed_shots and skip_failed:
print("Skipping failed idx shots %d" % idx)
continue
if idx in indexed_shots and skip_indexed:
print("Skipping already idx shots %d" % idx)
continue
refls_strong = flex.reflection_table.from_observations(El, spot_par)
if len(refls_strong) < min_spot_per_pattern:
print("Not enough spots shot %d, continuing!" % idx)
weak_shots.append(idx)
try:
np.savetxt(weak_shots_f, weak_shots, fmt="%d")
except:
pass
continue
# ==================================
# 2 color indexer of 2 color pattern
# ==================================
try:
orientAB = indexer_two_color(
reflections=refls_strong,
experiments=El,
params=mad_index_params)
orientAB.index()
except (Sorry, RuntimeError, AssertionError) as error:
print("####\nIndexingFailed T_T \n####")
print (error)
failed_shots.append(idx)
try:
np.savetxt(failed_idx_f, failed_shots, fmt="%d")
except:
pass
continue
print("####\n *_* IndexingWORKED %d *_* \n####" % idx)
n_idx += 1
indexed_shots.append(idx)
try:
np.savetxt(indexed_f, indexed_shots, fmt="%d")
except:
pass
crystalAB = orientAB.refined_experiments.crystals()[0]
refl_pkl = os.path.join(out_dir,refls_dir, "refl_%d_%s.pkl" % (idx, tag))
utils.save_flex(refls_strong, refl_pkl)
El[0].crystal = crystalAB
El_json = os.path.join(out_dir, Els_dir, "El_%d_%s.json" % (idx, tag))
El.as_json(filename=El_json)
if args.lastonly:
indices = [indices[-1]]
sgi = sgtbx.space_group_info(symbol)
sym = symmetry(unit_cell=ucell, space_group_info=sgi)
mset = miller.set(sym , flex_indices, anomalous_flag=True)
for i_iter in indices:
if six.PY3:
fcell_data = flex.double( np.load(data_files[i_iter], allow_pickle=True)['fvals'] )
else:
fcell_data = flex.double( np.load(data_files[i_iter])['fvals'] )
fobs = miller.array( mset, data=fcell_data).set_observation_type_xray_amplitude()
if args.savelast is not None and i_iter==indices[-1]:
utils.save_flex(fobs, args.savelast)
fobs_sel = fobs.select(fobs.resolution_filter_selection(d_max=30, d_min=args.dmin))
if args.binning:
r,c = utils.compute_r_factor_binned( ftruth, fobs_sel, verbose=True, d_max=30, d_min=args.dmin, n_bin=10)
r,c = utils.compute_r_factor( ftruth, fobs_sel, verbose=False, is_flex=True, sort_flex=True)
print ("Iter %d: Overall: Rtruth=%.4f, CCdeltaAnom=%.4f"% (i_iter, r,c))
else:
r,c = utils.compute_r_factor( ftruth, fobs_sel, verbose=False, is_flex=True, sort_flex=True)
print ("iter %d: Rtruth=%.4f, CCdeltaAnom=%.4f"% (i_iter, r,c))
print (fobs_sel.completeness())
if args.mtzoutput is not None:
fobs.as_mtz_dataset("Fobs").mtz_object().write(args.mtzoutput)
data2 = {
"dQ_AnotB": all_qAnotB,
"dQ_BnotA": all_qBnotA,
"dQ_AandB": all_qAandB,
"dQ_AorB": all_qAorB,
"dij_AnotB": all_dAnotB,
"dij_BnotA": all_dBnotA,
"dij_AandB": all_dAandB,
"dij_AorB": all_dAorB,
"res_AnotB": all_resAnotB,
"res_BnotA": all_resBnotA,
"res_AandB": all_resAandB,
"res_AorB": all_resAorB,
}
utils.save_flex(data2, odir + "/" + "run%d_details.pkl" % run)
dist_out3 = np.array(dist_out3)
cols = [
"numclose", "rmsd_v1", "fname", "run_num", "shot_idx", "frac_indexed",
"Nref", "Nindexed", "AnotB", "BnotA", "AandB", "AorB"
]
dtypes = [
np.int32, np.float32, str, np.int32, np.int32, np.float32, np.int32,
np.int32, np.int32, np.int32, np.int32, np.int32
]
df = pandas.DataFrame(dist_out3[:, [0] + range(2, 13)], columns=cols)
for i, col in enumerate(cols):
df[col] = df[col].astype(dtypes[i])
df.to_pickle(odir + "/" + "run%d_overview.pdpkl" % run)
default=None,
help="|F| reference pickle")
parser.add_argument(
"--referenceForCCano",
type=str,
default=None,
help=
"|F| reference pickle, for computing CCano (only heavy atoms should have anomalous scatering corrections in this reference)"
)
args = parser.parse_args()
from iotbx.reflection_file_reader import any_reflection_file
from cxid9114 import utils
F = any_reflection_file(args.input).as_miller_arrays()[0]
F = F.as_amplitude_array()
utils.save_flex(F, args.output)
print("wrote amplitudes from mtz as a cctbx miller array in file %s " %
args.output)
if args.reference is None:
exit()
# compare with ground truth
ftruth = utils.open_flex(args.reference) # for computing R with ground truth
fobs = F.select(F.resolution_filter_selection(d_max=30, d_min=2.125))
r, c = utils.compute_r_factor(ftruth,
fobs,
verbose=False,
is_flex=True,
sort_flex=True)
ftruth2 = utils.open_flex(args.referenceForCCano) # for comparing to CC ano
sad_index_params.indexing.stills.rmsd_min_px = 8
sad_index_params.indexing.debug = True
sad_index_params.indexing.fft1d.characteristic_grid = 0.029
#sad_index_params.indexing.refinement_protocol.mode = ""
# index two color pattern using fft1d
orient = indexer_base.from_parameters(
reflections=count_spots.as_single_shot_reflections(refls_strong,
inplace=False),
imagesets=[iset],
params=sad_index_params)
try:
orient.index()
except (Sorry, RuntimeError):
print("\n\n\t INDEXING FAILED\n")
failed_shots.append(idx)
np.savetxt(failed_idx_f, failed_shots, fmt="%d")
continue
indexed_shots.append(idx)
np.savetxt(indexed_f, indexed_shots, fmt="%d")
exp_name = os.path.join(outdir, "exp_%d.json" % Nprocessed)
refl_name = os.path.join(outdir, "refl_%d.pkl" % Nprocessed)
orient.export_as_json(orient.refined_experiments, file_name=exp_name)
utils.save_flex(orient.refined_reflections, refl_name)
crystals[idx] = orient.refined_experiments.crystals()[0]
Nprocessed += 1
utils.save_flex(crystals, os.path.join(outdir, "ssirp_cryst_r102.pkl"))
eB = Experiment()
eB.beam = beamB
eB.detector = detector
eB.imageset = iset
eB.crystal = cryst_model
EL = ExperimentList()
EL.append(eA)
EL.append(eB)
exp_json = os.path.join(out_dir, "expAB_%d.json" % idx)
reflA_pkl = os.path.join(out_dir, "reflA_%d.pkl" % idx)
reflB_pkl = os.path.join(out_dir, "reflB_%d.pkl" % idx)
orientAB.export_as_json(EL, exp_json)
utils.save_flex(reflsA, reflA_pkl)
utils.save_flex(reflsB, reflB_pkl)
dump_pkl = os.path.join(out_dir, "dump_%d.pkl" % idx)
t = loader.times[idx] # event time
sec, nsec, fid = t.seconds(), t.nanoseconds(), t.fiducial()
t_num, _ = utils.make_event_time(sec, nsec, fid)
dump = {
"crystalAB": crystalAB,
"event_time": t_num,
"tsec": sec,
"tnsec": nsec,
"tfid": fid,
#"res_opt": res_opt,
#"color_opt": color_opt,
beam=BeamA,
mask=mask)
dump = utils.open_flex("dump_%d.pkl" % shot_idx)
refls = dump['refls_strong']
spot_utils.as_single_shot_reflections(refls)
iset = dblock.extract_imagesets()[0]
if index:
orientAB = indexer_two_color(reflections=refls,
imagesets=[iset],
params=mad_index_params)
orientAB.index()
C = orientAB.refined_experiments.crystals()[0]
utils.save_flex({"C": C}, "C%d.pkl" % shot_idx)
from dxtbx.model.experiment_list import Experiment, ExperimentList
e = Experiment()
e.detector = D
e.beam = B
e.crystal = C
e.imageset = iset
el = ExperimentList()
el.append(e)
orientAB.export_as_json(el, "exp%d.json" % shot_idx)
##############
#############
##############
##############
##############
'refl': refl,
'optX': optX,
'optY': optY,
'optCrystal': optCrystal,
'hit_idx': hit_i,
'fcalc_f': fcalc_f,
'idx_proc_file': idx_proc_file
}
output_dir = os.path.dirname(output_prefix)
output_base = os.path.basename(output_prefix)
output_file = os.path.join(output_dir,
"good_hit_%d_%s" % (hit_i, output_base))
output_pkl = output_file + ".pkl"
utils.save_flex(output_dump, output_pkl)
if fine_scan:
results, fineX, fineY, fineA = fine_xyrefine(
output_pkl,
n_jobs=n_jobs,
use_weights=use_weights,
raw_image=raw_img)
#optA_fine = fineX * fineY * optX * optY * sqr(crystal.get_U()) * sqr(crystal.get_B())
optCrystal_fine = deepcopy(crystal)
optCrystal_fine.set_A(fineA)
output_dump['fine_scan_results'] = results
output_dump['optX_fine'] = fineX
output_dump['optY_fine'] = fineY
def run_paramList(Ntrials, odir, tag, rank, n_jobs, pkl_file):
import os
import sys
from copy import deepcopy
import numpy as np
import h5py
from IPython import embed
from scipy.ndimage.morphology import binary_dilation
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from dxtbx.model.crystal import CrystalFactory
from cctbx import crystal,crystal_orientation
from cxid9114 import utils
from cxid9114.sim import sim_utils
from cxid9114.spots import spot_utils
from cxid9114.bigsim.bigsim_geom import DET,BEAM
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.refine.jitter_refine import make_param_list
from cxid9114.bigsim import sim_spectra
from LS49.sim.step4_pad import microcrystal
data_pack = utils.open_flex(pkl_file)
CRYST = data_pack['crystalAB']
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True
Deff_A = 2200
length_um = 2.2
timelog = False
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
refls_strong = data_pack['refls_strong']
strong_mask_img = spot_utils.strong_spot_mask(
refls_strong, (1800,1800) )
# edge detection in the ground truth strong mask image
reference_img = (binary_dilation(strong_mask_img, iterations=1).astype(int) -
strong_mask_img.astype(int) ).astype(bool)
param_fileout = os.path.join( odir, "rank%d_%s.pkl" % (rank, tag))
param_list = make_param_list(
CRYST, DET, BEAM, Ntrials,
rot=0.09, cell=0.1, eq=(1,1,0),
min_Ncell=20, max_Ncell=40,
min_mos_spread=0.005, max_mos_spread=0.02)
for p in param_list:
print(p['crystal'].get_unit_cell().parameters())
shot_idx = int(data_pack['img_f'].split("_")[-1].split(".")[0])
Fluxes = spec_data[2] #shot_idx]
Pmax = param_list[0]
F1max = 0
for i_trial in range(Ntrials):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Trial %d / %d" % (rank, i_trial+1, Ntrials))
print ("<><><><><><><><><><><><><><>")
if (rank==0 and i_trial % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
assert (len(wave_chans)==len(Fluxes)==len(sfall_main))
if np.sum(Fluxes)==0:
print ("Cannot simulate with an all-zeros spectrum!")
sys.exit()
N = crystal.number_of_cells(sfall_main[0].unit_cell())
Ncells_abc = (N,N,N)
if force_twocolor:
Fluxes *= 0
Fluxes[ilow] = 1e12
Fluxes[ihigh]=1e12
P = param_list[i_trial]
simsAB = sim_utils.sim_twocolors2(
P['crystal'],
DET,
BEAM,
sfall_main,
en_chans,
Fluxes,
pids = None,
profile="gauss",
oversample=0,
Ncells_abc = Ncells_abc, #P['Ncells_abc'],
mos_dom=mos_doms,
verbose=verbose,
mos_spread=mos_spread_deg,
#@mos_spread=P['mos_spread'],
cuda=True,
device_Id=rank,
beamsize_mm=beamsize_mm,
exposure_s=exposure_s,
boost=crystal.domains_per_crystal)
out = np.sum( [ simsAB[i][0] for i in simsAB.keys() if simsAB[i]], axis=0)
if out.shape==():
print("This simsAB output has an empty shape, something is wrong!")
sys.exit()
trial_refls = spot_utils.refls_from_sims([out], DET, BEAM, thresh=thresh)
trial_spotmask = spot_utils.strong_spot_mask(
trial_refls, (1800,1800) )
trial_img = (binary_dilation(trial_spotmask, iterations=1).astype(int) -
trial_spotmask.astype(int) ).astype(bool)
comp_img = trial_img.astype(int) + reference_img.astype(int)*2
Nfalse_neg = (comp_img==2).sum() # reference has signal but trial doesnt
Nfalse_pos = (comp_img==1).sum() # trial has signal but reference doesnt
Ntrue_pos = (comp_img==3).sum() #
Precision = float(Ntrue_pos) / (Ntrue_pos + Nfalse_pos)
Recall = float(Ntrue_pos) / (Ntrue_pos + Nfalse_neg)
F1 = 2.*(Precision*Recall) / (Precision+Recall)
if F1 > F1max:
Pmax = {'crystal': P['crystal'],
'mos_spread': P['mos_spread'],
'Ncells_abc': P['Ncells_abc'], "F1": F1}
F1max = F1
print("Rank %d, Trial %d: F1score = %.5f" % (rank, i_trial, F1))
utils.save_flex(Pmax, param_fileout)
return F1max
bad_idx = u_all_Hi.index((0, 0, 0))
u_all_Hi.pop(bad_idx)
u_all_amp.pop(bad_idx)
indices = flex.miller_index(u_all_Hi)
data = flex.double(u_all_amp)
# save as an MTZ
mset = miller.set(symm, indices=indices, anomalous_flag=True)
marray = miller.array(mset, data).set_observation_type_xray_amplitude()
if args.mtzoutput:
mtz = marray.as_mtz_dataset(column_root_label='fobs',
wavelength=parameters.WAVELEN_HIGH)
ob = mtz.mtz_object()
ob.write(args.out)
# save as a PKL
else:
utils.save_flex(marray, args.out)
if args.truth is not None:
Ftruth = utils.open_flex(args.truth)
marray_2 = marray.select(
marray.resolution_filter_selection(d_max=30, d_min=args.dhighres))
r, c = utils.compute_r_factor(Ftruth,
marray_2,
is_flex=True,
sort_flex=True)
print("Truth R-factor=%4f and CC-Delta-anom=%.4f out to %.4f Angstrom" %
(r, c, args.dhighres))
iset = loader.get_imageset(loader.get_image_file())
dblock = DataBlockFactory.from_imageset(iset[idx:idx + 1])[0]
refl = flex.reflection_table.from_observations(dblock, find_spot_params)
imgs.append(img)
refls.append(refl)
info_fname = image_fname.replace(".h5", ".pkl")
sim_data = utils.open_flex(info_fname)
orient = indexer_two_color(
reflections=count_spots.as_single_shot_reflections(refl,
inplace=False),
imagesets=[iset],
params=indexing_params)
try:
orient.index()
crystals = [o.crystal for o in orient.refined_experiments]
rmsd = orient.best_rmsd
sim_data["sim_indexed_rmsd"] = rmsd
sim_data["sim_indexed_crystals"] = crystals
sim_data["sim_indexed_refls"] = refl
except:
print("FAILED!")
utils.save_flex(sim_data, info_fname + out_tag)
images_and_refls_to_simview(output_pref, imgs, refls)
if np.any(kept_sel):
kept_refls = reflsPP[pid].select(flex.bool(kept_sel))
spot_masks = spot_utils.strong_spot_mask(kept_refls,
img_sh,
as_composite=False)
# color the kept reflections green!
winner_patches = get_spot_patches(spot_masks, ec='C2')
for p in winner_patches:
ax.add_patch(p)
ax.set_title("Panel %d; Some Winners!" % pid)
# save the figure
figdir = os.path.join(outdir, "panel%d" % pid)
if not os.path.exists(figdir):
os.makedirs(figdir)
figname = "pan%d_trial%d.jpg" % (pid, Ntrial)
figpath = os.path.join(figdir, figname)
plt.savefig(figpath, dpi=50)
while ax.patches:
ax.patches.pop()
dump_name = outputname.replace(".pkl", "_probable_refls.pkl")
utils.save_flex(good_refls, dump_name)
if save_sims:
h5.close()
except:
pass
crystalAB = orientAB.refined_experiments.crystals()[0]
E = Experiment()
E.beam = BEAM
E.detector = DET
E.imageset = iset
E.crystal = crystalAB
EL = ExperimentList()
EL.append(E)
exp_json = os.path.join(out_dir, "exp_%d_%s.json" % (idx, tag) )
refl_pkl = os.path.join(out_dir, "refl_%d_%s.pkl" % (idx, tag))
orientAB.export_as_json( EL, exp_json)
utils.save_flex(refls_strong, refl_pkl)
t = loader.times[idx] # event time
sec, nsec, fid = t.seconds(), t.nanoseconds(), t.fiducial()
t_num, _ = utils.make_event_time( sec, nsec, fid)
dump = {"crystalAB": crystalAB,
"event_time": t_num,
"tsec": sec,"tnsec": nsec, "tfid": fid,
"beamA": beamA,
"beamB": beamB,
"detector": detector,
"rmsd": orientAB.best_rmsd,
"refls_strong": refls_strong}
dump_pkl= os.path.join(out_dir, "dump_%d_%s.pkl" % (idx, tag))
a=ucell_a,
b=ucell_b,
c=ucell_c)
# for the non-outlier reflections, store the overlap values for each trial
# then we can do a global clustering to see if there is a preferred simulation parameter
data_dict["scores"] = list(map(list, overlaps[winners].T))
Amats = [list(C.get_A()) for C in new_crystals]
data_dict["Amat"] = Amats
df = pandas.DataFrame(data_dict)
df['ave_score'] = np.vstack(df.scores.values).mean(1)
df.to_pickle(outputname)
idx_max = df.ave_score.idxmax()
best_model = df.iloc[idx_max].drop(['scores', "Amat"])
best_Amat = tuple(df.iloc[idx_max]['Amat'])
with open(outputname.replace(".pkl", "_best.txt"), "w") as o:
o.write(best_model.to_string())
dump_name = outputname.replace(".pkl", "_best_dump.pkl")
best_cryst = deepcopy(new_crystals[0])
best_cryst.set_A(best_Amat)
utils.save_flex({
"good_refls": good_refls,
"best_cryst": best_cryst
}, dump_name)
'direction': (0.0, 0.0, 1.0),
'divergence': 0.0,
'flux': 1e12,
'polarization_fraction': 1.,
'polarization_normal': (0.0, 1.0, 0.0),
'sigma_divergence': 0.0,
'transmission': 1.0,
'wavelength': 1.4
}
# dxtbx crystal description
hall_sym = sgtbx.space_group_info(
number=19).type().hall_symbol() # ' P 2ac 2ab'
cryst_descr = {
'__id__': 'crystal',
'real_space_a': (127.7, 0, 0),
'real_space_b': (0, 225.4, 0),
'real_space_c': (0, 0, 306.1),
'space_group_hall_symbol': hall_sym
}
beam = BeamFactory.from_dict(beam_descr)
whole_det = DetectorFactory.from_dict(cspad)
cryst = CrystalFactory.from_dict(cryst_descr)
from cxid9114 import utils
utils.save_flex(whole_det, "ps2.det.pkl")
utils.save_flex(cryst, "ps2.cryst.pkl")
utils.save_flex(beam, "ps2.beam.pkl")
hkl_map2 = {v: k for k, v in hkl_map.iteritems()}
Nhkl = len(hkl_map)
assert (Nh == Nhkl)
hout, Iout = [], []
for i in range(Nhkl):
h = hkl_map2[i]
val = IA[i]
hout.append(h)
Iout.append(np.exp(val))
sg = sgtbx.space_group(" P 4nw 2abw")
Symm = crystal.symmetry(unit_cell=(79, 79, 38, 90, 90, 90), space_group=sg)
hout = tuple(hout)
mil_idx = flex.miller_index(hout)
mil_set = miller.set(crystal_symmetry=Symm,
indices=mil_idx,
anomalous_flag=True)
Iout_flex = flex.double(np.ascontiguousarray(Iout))
mil_ar = miller.array(mil_set,
data=Iout_flex).set_observation_type_xray_intensity()
utils.save_flex(mil_ar, "n00b_begin.pkl")
waveA = ENERGY_CONV / 8944.
waveA = ENERGY_CONV / 9034
IA = mil_ar
out = IA.as_mtz_dataset(column_root_label="Iobs", title="B", wavelength=waveA)
out.add_miller_array(miller_array=IA.average_bijvoet_mates(),
column_root_label="IMEAN")
obj = out.mtz_object()
obj.write(args.o)
beamA,
FF,
ENERGIES,
FLUX,
pids=None,
profile='tophat', # new_shape['shape'],
oversample=0, # this should let nanoBragg decide how to oversample!
Ncells_abc=new_shape['Ncells_abc'],
mos_dom=Nmos_dom,
mos_spread=0, #new_shape['mos_spread'],
roi_pp=roi_pp,
counts_pp=counts_pp,
cuda=False)
JFC = 10
sim_imgs = np.array(simsAB[0]) + np.array(simsAB[1])
h5.create_dataset("sim_imgs_%d" % i_trial, data=sim_imgs)
print("\t... took %.4f seconds" % (time.time() - t))
Ncells_abc.append(new_shape['Ncells_abc'])
mos_doms.append(Nmos_dom)
mos_spread.append(new_shape['mos_spread'])
xtal_shapes.append(new_shape['shape'])
a, b, c, _, _, _ = new_crystal.get_unit_cell().parameters()
ucell_a.append(a)
ucell_b.append(b)
ucell_c.append(c)
new_crystals.append(new_crystal)
utils.save_flex(refls_strong, outputname + "_refls.pkl")
Ncells_abc=(7, 7, 7), mos_dom=20, mos_spread=0.0)
refl_simA_old = spot_utils.refls_from_sims(simsAB_old[0], detector, beamA)
refl_simB_old = spot_utils.refls_from_sims(simsAB_old[1], detector, beamB)
residA_old = metrics.check_indexable(
refls_strong, refl_simA_old, detector, beamA, crystalAB, hkl_tol)
residB_old = metrics.check_indexable(
refls_strong, refl_simB_old, detector, beamB, crystalAB, hkl_tol)
dump = {"crystalAB": crystalAB,
"optCrystal": optCrystal,
"residA": residA,
"residB": residB,
"residA_old": residA_old,
"residB_old": residB_old,
"beamA": beamA,
"beamB": beamB,
"overlap": overlap,
"detector": detector,
"refls_simA": refl_simA,
"refls_simB": refl_simB,
"refls_simA_old": refl_simA_old,
"refls_simB_old": refl_simB_old,
"rmsd_v1": data['rmsd'],
"refls_strong": refls_strong}
dump_name = data_name.replace(".pkl", "_%s.pkl" % tag)
utils.save_flex(dump, dump_name)
print "Wrote %s" % dump_name
data[h]["sigB"] = sigB
can_analyze_spec = sigA > thresh or sigB > thresh
if can_analyze_spec:
if sigA < 0:
sigA = 0.0001
if sigB < 0:
sigB = 0.0001
fracA, fracB = sigA / (sigA + sigB), sigB / (sigA + sigB)
data[h]["fracA"] = fracA
data[h]["fracB"] = fracB
data[h]['can_analyze'] = True
print "Shot %d has 2 color fractions %f" % (h, fracA)
else:
data[h]['fracA'] = 0.5 # NOTE: not sure what to do here..
data[h]['fracB'] = 0.5
data[h]['can_analyze'] = False
utils.save_flex(data, output_name)
N_has_spec = np.sum([1 for k in data if data[k]['spectrum'] is not None])
N_weak_spec = np.sum([
1 for k in data
if not data[k]['can_analyze'] and data[k]['spectrum'] is not None
])
print("%d / %d hits had spectrum data!" % (N_has_spec, Ndata))
print("%d / %d hits had spectrum data but it was too weak!" %
(N_weak_spec, N_has_spec))
print("Saved pickle %s. Done. " % output_name)
