def xyscan(crystal, fcalcs_data, fracA, fracB, strong, rotxy_series, jid):
"""
:param crystal_file:
:param energies:
:param fcalcs_at_en:
:param fracA:
:param fracB
:param refl_file:
:param rotxy_series:
:return:
"""
#crystal = utils.open_flex(crystal_file)
#strong_spots = utils.open_flex(refl_file)
energies, fcalcs_at_en = fcalcs_data['energy'], fcalcs_data["fcalc"]
Patts = PatternFactory()
Patts.adjust_mosaicity(2, 0.05) # defaults
flux_per_en = [fracA * 1e14, fracB * 1e14]
img_size = Patts.detector.to_dict()['panels'][0]['image_size']
found_spot_mask = spot_utils.strong_spot_mask(strong, img_size)
overlaps = []
for rX, rY in rotxy_series:
sim_patt = Patts.make_pattern2(crystal=deepcopy(crystal),
flux_per_en=flux_per_en,
energies_eV=energies,
fcalcs_at_energies=fcalcs_at_en,
mosaic_spread=None,
mosaic_domains=None,
ret_sum=True,
Op=rX * rY)
sim_sig_mask = sim_patt > 0
overlaps.append(np.sum(sim_sig_mask * found_spot_mask))
print("JOBOBOBOBO %d" % jid)
return overlaps
def integrate(R, badmask, data, gain=28, fit_bg=True):
"""
get the crystal scatter, background scatter, and photon counting noise
for the reflections listed in the table R
:param R: reflection table
:param badmask: mask in numpy format, same shape as data
:param data: data
:param gain: detector gain per photon
:return: 3 arrays, one for signal, background and noise
"""
Nrefl = len(R)
Rpp = spot_utils.refls_by_panelname(
R
) # this is a dictionary whose key (0-63) unlock that panels reflections
allspotmask = {}
bgtilt = {}
for pid in Rpp:
# load the spot mask for all strong spots for this panel
allspotmask[pid] = spot_utils.strong_spot_mask(Rpp[pid], (185, 194))
if fit_bg:
bgtilt[pid], _, _ = tilting_plane(data[pid],
mask=(~allspotmask[pid]) *
badmask[pid],
zscore=8)
signa = np.zeros(Nrefl)
bg = np.zeros(Nrefl)
noise = np.zeros_like(signa)
for i_r, refl in enumerate(R):
pid = refl['panel']
spotmask = refl['shoebox'].mask.as_numpy_array() == 5
f1, f2, s1, s2, _, _ = refl[
'shoebox'].bbox # fast scan and slow scan edges of bounding box
thisspotmask = np.zeros_like(allspotmask[pid])
thisspotmask[s1:s2, f1:f2] = spotmask
#smask = (thisspotmask ^ allspotmask[pid])
signa[i_r] = data[pid][thisspotmask].sum()
if fit_bg:
bg[i_r] = bgtilt[pid][thisspotmask].sum()
# else bg[i_r] is going to remain 0
signa[i_r] = (signa[i_r] - bg[i_r]) / gain
noise[i_r] = np.sqrt(signa[i_r])
return signa, bg, noise
def refine_cell(data):
spot_mask = spot_utils.strong_spot_mask(data['refl'], (1800, 1800))
Patts = sim_utils.PatternFactory()
Patts.adjust_mosaicity(2, 0.5)
energy, fcalc = sim_utils.load_fcalc_file(data['fcalc_f'])
crystal = data['crystal']
a, b, c, _, _, _ = crystal.get_unit_cell().parameters()
optX, optY = data['optX'], data['optY']
optX_fine, optY_fine = data['optX_fine'], data['optY_fine']
Op = optX_fine * optY_fine * optX * optY
crystal.set_U(Op)
overlaps = []
imgs_all = []
percs = np.arange(-0.005, 0.006, 0.001)
crystals = []
for i in percs:
for j in percs:
crystal2 = deepcopy(crystal)
a2 = a + a * i
c2 = c + c * j
B2 = sqr((a2, 0, 0, 0, a2, 0, 0, 0, c2)).inverse()
crystal2.set_B(B2)
sim_patt = Patts.make_pattern2(
crystal=crystal2,
flux_per_en=[data['fracA'] * 1e14, data['fracB'] * 1e14],
energies_eV=energy,
fcalcs_at_energies=fcalc,
mosaic_spread=None,
mosaic_domains=None,
ret_sum=True,
Op=None)
sim_sig_mask = sim_patt > 0
overlaps.append(np.sum(sim_sig_mask * spot_mask))
crystals.append(deepcopy(crystal2))
imgs_all.append(sim_patt)
refls_all = [data["refl"]] * len(imgs_all)
utils.images_and_refls_to_simview("cell_refine", imgs_all, refls_all)
return overlaps, crystals
def __init__(self, crystal, Patt, refls, data_image, szx=30, szy=30):
"""
Some tools for jittering the indexing solution
to try and come up with a better solution
THis uses simtbx wrapped into the Patt object
:param crystal: dxtbx crystal
:param Patt: an instance of PatternFactory that has been
primed (Patt.primer has been run)
Note, primer sets the Amatrix, Fcalcs, energy, and flux
:param refls: strong spots, we will simulate there vicinity
and use the simultion overlap as a proxy for indexing
solution quality
:param data_image: the raw data image that is used to compute
overlap
"""
detector = Patt.detector[Patt.panel_id]
self.FS = detector.get_fast_axis()
self.SS = detector.get_slow_axis()
self.A = sqr(crystal.get_A())
self.U = sqr(crystal.get_U())
self.B = sqr(crystal.get_B())
self.s0 = Patt.beam.get_s0()
self.ucell_abc = list(crystal.unit_cell().parameters())[:3]
self.crystal = crystal
self.Patt = Patt
self.data = data_image
self.rois = spot_utils.get_spot_roi(
refls,
dxtbx_image_size=detector.get_image_size(),
szx=szx,
szy=szy)
self.spotX, self.spotY, _ = spot_utils.xyz_from_refl(refls)
self.spot_mask = spot_utils.strong_spot_mask(refls, self.data.shape)
self.counts = spot_utils.count_roi_overlap(self.rois,
img_size=self.data.shape)
if np.any(self.counts > 0):
self.should_norm = True
else:
self.should_norm = False
Malls = {}
for pid, img in izip(pids, pan_imgs):
panel = detector[pid]
rois = spot_utils.get_spot_roi(reflsPP[pid],
dxtbx_image_size=panel.get_image_size(),
szx=szx,
szy=szy)
counts = spot_utils.count_roi_overlap(rois, img_size=img.shape)
roi_pp.append(rois)
counts_pp.append(counts)
spot_masks = spot_utils.strong_spot_mask(reflsPP[pid],
img_sh,
as_composite=False)
# composite mask
Mall = np.any(spot_masks, axis=0)
yall, xall = np.where(Mall) # use comp mask to determine the boundary
xlims[pid] = (xall.min() - szx, xall.max() + szx)
ylims[pid] = (yall.max() + szy, yall.min() - szy)
patches[pid] = get_spot_patches(spot_masks)
Malls[pid] = spot_masks
pan_img_idx = {pid: idx for idx, pid in enumerate(pids)}
def xyscan(crystal,
fcalcs_energies,
fcalcs,
fracA,
fracB,
strong,
rotxy_series,
jid,
mos_dom=1,
mos_spread=0.05,
flux=1e14,
use_weights=False,
raw_image=None):
"""
:param crystal:
:param fcalcs_energies:
:param fcalcs:
:param fracA:
:param fracB:
:param strong:
:param rotxy_series:
:param jid:
:param mos_dom:
:param mos_spread:
:param flux:
:param use_weights:
:return:
"""
Patts = sim_utils.PatternFactory()
Patts.adjust_mosaicity(mos_dom, mos_spread) # defaults
flux_per_en = [fracA * flux, fracB * flux]
img_size = Patts.detector.to_dict()['panels'][0]['image_size']
found_spot_mask = spot_utils.strong_spot_mask(strong, img_size)
#if use_weights:
# if raw_image is None:
# raise ValueError("Cant use weights if raw image is None")
# spot_signals = raw_image * found_spot_mask
# weights = spot_signals / spot_signals.max()
overlaps = []
for rots in rotxy_series:
if len(rots) == 2:
Op = rots[0] * rots[1]
elif len(rots) == 3:
Op = rots[0] * rots[1] * rots[2]
elif len(rots == 1):
Op = rots[0]
else:
raise ValueError("rotxy_series should be list of 1,2, or 3-tuples")
sim_patt = Patts.make_pattern2(crystal=deepcopy(crystal),
flux_per_en=flux_per_en,
energies_eV=fcalcs_energies,
fcalcs_at_energies=fcalcs,
mosaic_spread=None,
mosaic_domains=None,
ret_sum=True,
Op=Op)
if use_weights:
dblock = utils.datablock_from_numpyarrays(image=sim_patt,
detector=Patts.detector,
beam=Patts.beam)
sim_refl = flex.reflection_table.from_observations(
dblock, params=find_spot_params)
sim_spot_mask = spot_utils.strong_spot_mask(
sim_refl, sim_patt.shape)
overlaps.append(np.sum(sim_spot_mask * found_spot_mask))
#weights2 = sim_patt / sim_patt.max()
#overlaps.append( np.sum(sim_sig_mask * found_spot_mask * weights * weights2))
else:
sim_sig_mask = sim_patt > 0
overlaps.append(np.sum(sim_sig_mask * found_spot_mask))
print "JOB %d" % jid
return overlaps
def run_paramList(Ntrials, odir, tag, rank, n_jobs, pkl_file):
import os
import sys
from copy import deepcopy
import numpy as np
import h5py
from IPython import embed
from scipy.ndimage.morphology import binary_dilation
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from dxtbx.model.crystal import CrystalFactory
from cctbx import crystal,crystal_orientation
from cxid9114 import utils
from cxid9114.sim import sim_utils
from cxid9114.spots import spot_utils
from cxid9114.bigsim.bigsim_geom import DET,BEAM
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.refine.jitter_refine import make_param_list
from cxid9114.bigsim import sim_spectra
from LS49.sim.step4_pad import microcrystal
data_pack = utils.open_flex(pkl_file)
CRYST = data_pack['crystalAB']
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True
Deff_A = 2200
length_um = 2.2
timelog = False
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
refls_strong = data_pack['refls_strong']
strong_mask_img = spot_utils.strong_spot_mask(
refls_strong, (1800,1800) )
# edge detection in the ground truth strong mask image
reference_img = (binary_dilation(strong_mask_img, iterations=1).astype(int) -
strong_mask_img.astype(int) ).astype(bool)
param_fileout = os.path.join( odir, "rank%d_%s.pkl" % (rank, tag))
param_list = make_param_list(
CRYST, DET, BEAM, Ntrials,
rot=0.09, cell=0.1, eq=(1,1,0),
min_Ncell=20, max_Ncell=40,
min_mos_spread=0.005, max_mos_spread=0.02)
for p in param_list:
print(p['crystal'].get_unit_cell().parameters())
shot_idx = int(data_pack['img_f'].split("_")[-1].split(".")[0])
Fluxes = spec_data[2] #shot_idx]
Pmax = param_list[0]
F1max = 0
for i_trial in range(Ntrials):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Trial %d / %d" % (rank, i_trial+1, Ntrials))
print ("<><><><><><><><><><><><><><>")
if (rank==0 and i_trial % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
assert (len(wave_chans)==len(Fluxes)==len(sfall_main))
if np.sum(Fluxes)==0:
print ("Cannot simulate with an all-zeros spectrum!")
sys.exit()
N = crystal.number_of_cells(sfall_main[0].unit_cell())
Ncells_abc = (N,N,N)
if force_twocolor:
Fluxes *= 0
Fluxes[ilow] = 1e12
Fluxes[ihigh]=1e12
P = param_list[i_trial]
simsAB = sim_utils.sim_twocolors2(
P['crystal'],
DET,
BEAM,
sfall_main,
en_chans,
Fluxes,
pids = None,
profile="gauss",
oversample=0,
Ncells_abc = Ncells_abc, #P['Ncells_abc'],
mos_dom=mos_doms,
verbose=verbose,
mos_spread=mos_spread_deg,
#@mos_spread=P['mos_spread'],
cuda=True,
device_Id=rank,
beamsize_mm=beamsize_mm,
exposure_s=exposure_s,
boost=crystal.domains_per_crystal)
out = np.sum( [ simsAB[i][0] for i in simsAB.keys() if simsAB[i]], axis=0)
if out.shape==():
print("This simsAB output has an empty shape, something is wrong!")
sys.exit()
trial_refls = spot_utils.refls_from_sims([out], DET, BEAM, thresh=thresh)
trial_spotmask = spot_utils.strong_spot_mask(
trial_refls, (1800,1800) )
trial_img = (binary_dilation(trial_spotmask, iterations=1).astype(int) -
trial_spotmask.astype(int) ).astype(bool)
comp_img = trial_img.astype(int) + reference_img.astype(int)*2
Nfalse_neg = (comp_img==2).sum() # reference has signal but trial doesnt
Nfalse_pos = (comp_img==1).sum() # trial has signal but reference doesnt
Ntrue_pos = (comp_img==3).sum() #
Precision = float(Ntrue_pos) / (Ntrue_pos + Nfalse_pos)
Recall = float(Ntrue_pos) / (Ntrue_pos + Nfalse_neg)
F1 = 2.*(Precision*Recall) / (Precision+Recall)
if F1 > F1max:
Pmax = {'crystal': P['crystal'],
'mos_spread': P['mos_spread'],
'Ncells_abc': P['Ncells_abc'], "F1": F1}
F1max = F1
print("Rank %d, Trial %d: F1score = %.5f" % (rank, i_trial, F1))
utils.save_flex(Pmax, param_fileout)
return F1max
dump_file = "%s/results/run%d/dump_%d_feb8th.pkl" % (root, run, idx)
f = h5py.File(
"%s/videos/run%d/shot%d%s/dump_%d_feb8th_jitt.h5py" %
(root, run, idx, tag, idx), "r")
pids = f['pids'].value
#pan_imgs = f['pan_imgs'].value
# for the simulated images
keys = [k for k in f.keys() if k.startswith("sim_imgs_")]
Ntrials = len(keys)
d = utils.open_flex(dump_file)
R = d['refls_strong']
Rpp = spot_utils.refls_by_panelname(R)
Masks = spot_utils.strong_spot_mask(R, (185, 194), as_composite=False)
min_prob = 0.1 / Ntrials
Ntotal_bad = Ntotal_spots = 0
for pidx, pid in enumerate(pids):
sim = np.array([f[k][pidx] for k in keys])
im = sim.mean(0)
im /= im.max() # make it a probability
M0 = spot_utils.strong_spot_mask(Rpp[pid], (185, 194), as_composite=False)
vals = []
for m in M0:
vec = m * im
val = vec[vec > 0].mean()
def integrate2(R, badmask, data, gain=28, fit_bg=True, zscore=8, sz=8):
"""
get the crystal scatter, background scatter, and photon counting noise
for the reflections listed in the table R
:param R: reflection table
:param badmask: mask in numpy format, same shape as data
:param data: data
:param gain: detector gain per photon
:return: 3 arrays, one for signal, background and noise
"""
from dials.algorithms.shoebox import MaskCode
fg_code = MaskCode.Foreground.real
Nrefl = len(R)
fs_dim = 194
ss_dim = 185
Rpp = spot_utils.refls_by_panelname(
R
) # this is a dictionary whose key (0-63) unlock that panels reflections
allspotmask = {}
for pid in Rpp:
# load the spot mask for all strong spots for this panel
allspotmask[pid] = spot_utils.strong_spot_mask(Rpp[pid],
(ss_dim, fs_dim))
signa = np.zeros(Nrefl)
bg = np.zeros(Nrefl)
noise = np.zeros_like(signa)
pix_per = np.zeros(Nrefl, int)
for i_r, refl in enumerate(R):
pid = refl['panel']
spotmask = refl['shoebox'].mask.as_numpy_array() & fg_code == fg_code
f1, f2, s1, s2, _, _ = refl[
'shoebox'].bbox # fast scan and slow scan edges of bounding box
icent, jcent, _ = refl['xyzobs.px.value']
thisspotmask = np.zeros_like(allspotmask[pid])
thisspotmask[s1:s2, f1:f2] = spotmask
i1 = int(max(icent - .5 - sz, 0))
i2 = int(min(icent - .5 + sz, fs_dim))
j1 = int(max(jcent - .5 - sz, 0))
j2 = int(min(jcent - .5 + sz, ss_dim))
sub_data = data[pid][j1:j2, i1:i2]
sub_mask = ((~allspotmask[pid]) * badmask[pid])[j1:j2, i1:i2]
sub_thisspotmask = thisspotmask[j1:j2, i1:i2]
Is = sub_data[sub_thisspotmask].sum()
if fit_bg:
tilt, bgmask, coeff = tilting_plane(sub_data,
mask=sub_mask,
zscore=zscore)
bg_fit_mask = np.logical_and(~bgmask, sub_mask)
m = sub_thisspotmask.sum()
n = bg_fit_mask.sum()
m2n = float(m) / float(
n
) # ratio of number of background to number of strong spot pixels
# modifuf Is according to background plane fit
Is = Is - tilt[sub_thisspotmask].sum()
# store background pix according to Leslie 99
Ibg = m2n * sub_data[bg_fit_mask].sum()
else:
Ibg = 0
signa[i_r] = Is # signal in the spot
bg[i_r] = Ibg # background around the spot
noise[i_r] = (Is + Ibg + m2n * Ibg) / gain
pix_per[i_r] = thisspotmask.sum()
return signa, bg, noise, pix_per
os.path.join(indir, f) for f in os.listdir(indir) if f.endswith(".pkl")
][0]
print h5_name, pkl_name
f = h5py.File(h5_name, "r")
D = utils.open_flex(pkl_name)
print "Using %d reflections!" % len(D)
Dpp = spot_utils.refls_by_panelname(D)
pids = f['pids'].value
pid_map = {p: i for i, p in enumerate(pids)}
Nsim = sum([1 for k in f.keys() if k.startswith('sim_imgs')])
dsets = [f['sim_imgs_%d' % x] for x in range(Nsim)]
Dpp_pids = Dpp.keys()
strong_mask = array(
[spot_utils.strong_spot_mask(Dpp[p], img_sh) for p in Dpp_pids])
thresh = 2 # 2 photon threshold
scores = {}
for i, mask in enumerate(strong_mask):
pid = Dpp_pids[i]
dset_idx = pid_map[pid]
pan = array([d[dset_idx] > thresh for d in dsets])
scores[pid] = [(~logical_xor(mask, p)).sum() for p in pan]
vals = []
for p in Dpp_pids:
s = scores[p]
mx = max(s)
idx = where(array(s) == mx)[0]
vals += list(idx)
