def xyscan(crystal, fcalcs_data, fracA, fracB, strong, rotxy_series, jid): """ :param crystal_file: :param energies: :param fcalcs_at_en: :param fracA: :param fracB :param refl_file: :param rotxy_series: :return: """ #crystal = utils.open_flex(crystal_file) #strong_spots = utils.open_flex(refl_file) energies, fcalcs_at_en = fcalcs_data['energy'], fcalcs_data["fcalc"] Patts = PatternFactory() Patts.adjust_mosaicity(2, 0.05) # defaults flux_per_en = [fracA * 1e14, fracB * 1e14] img_size = Patts.detector.to_dict()['panels'][0]['image_size'] found_spot_mask = spot_utils.strong_spot_mask(strong, img_size) overlaps = [] for rX, rY in rotxy_series: sim_patt = Patts.make_pattern2(crystal=deepcopy(crystal), flux_per_en=flux_per_en, energies_eV=energies, fcalcs_at_energies=fcalcs_at_en, mosaic_spread=None, mosaic_domains=None, ret_sum=True, Op=rX * rY) sim_sig_mask = sim_patt > 0 overlaps.append(np.sum(sim_sig_mask * found_spot_mask)) print("JOBOBOBOBO %d" % jid) return overlaps
def integrate(R, badmask, data, gain=28, fit_bg=True): """ get the crystal scatter, background scatter, and photon counting noise for the reflections listed in the table R :param R: reflection table :param badmask: mask in numpy format, same shape as data :param data: data :param gain: detector gain per photon :return: 3 arrays, one for signal, background and noise """ Nrefl = len(R) Rpp = spot_utils.refls_by_panelname( R ) # this is a dictionary whose key (0-63) unlock that panels reflections allspotmask = {} bgtilt = {} for pid in Rpp: # load the spot mask for all strong spots for this panel allspotmask[pid] = spot_utils.strong_spot_mask(Rpp[pid], (185, 194)) if fit_bg: bgtilt[pid], _, _ = tilting_plane(data[pid], mask=(~allspotmask[pid]) * badmask[pid], zscore=8) signa = np.zeros(Nrefl) bg = np.zeros(Nrefl) noise = np.zeros_like(signa) for i_r, refl in enumerate(R): pid = refl['panel'] spotmask = refl['shoebox'].mask.as_numpy_array() == 5 f1, f2, s1, s2, _, _ = refl[ 'shoebox'].bbox # fast scan and slow scan edges of bounding box thisspotmask = np.zeros_like(allspotmask[pid]) thisspotmask[s1:s2, f1:f2] = spotmask #smask = (thisspotmask ^ allspotmask[pid]) signa[i_r] = data[pid][thisspotmask].sum() if fit_bg: bg[i_r] = bgtilt[pid][thisspotmask].sum() # else bg[i_r] is going to remain 0 signa[i_r] = (signa[i_r] - bg[i_r]) / gain noise[i_r] = np.sqrt(signa[i_r]) return signa, bg, noise
def refine_cell(data): spot_mask = spot_utils.strong_spot_mask(data['refl'], (1800, 1800)) Patts = sim_utils.PatternFactory() Patts.adjust_mosaicity(2, 0.5) energy, fcalc = sim_utils.load_fcalc_file(data['fcalc_f']) crystal = data['crystal'] a, b, c, _, _, _ = crystal.get_unit_cell().parameters() optX, optY = data['optX'], data['optY'] optX_fine, optY_fine = data['optX_fine'], data['optY_fine'] Op = optX_fine * optY_fine * optX * optY crystal.set_U(Op) overlaps = [] imgs_all = [] percs = np.arange(-0.005, 0.006, 0.001) crystals = [] for i in percs: for j in percs: crystal2 = deepcopy(crystal) a2 = a + a * i c2 = c + c * j B2 = sqr((a2, 0, 0, 0, a2, 0, 0, 0, c2)).inverse() crystal2.set_B(B2) sim_patt = Patts.make_pattern2( crystal=crystal2, flux_per_en=[data['fracA'] * 1e14, data['fracB'] * 1e14], energies_eV=energy, fcalcs_at_energies=fcalc, mosaic_spread=None, mosaic_domains=None, ret_sum=True, Op=None) sim_sig_mask = sim_patt > 0 overlaps.append(np.sum(sim_sig_mask * spot_mask)) crystals.append(deepcopy(crystal2)) imgs_all.append(sim_patt) refls_all = [data["refl"]] * len(imgs_all) utils.images_and_refls_to_simview("cell_refine", imgs_all, refls_all) return overlaps, crystals
def __init__(self, crystal, Patt, refls, data_image, szx=30, szy=30): """ Some tools for jittering the indexing solution to try and come up with a better solution THis uses simtbx wrapped into the Patt object :param crystal: dxtbx crystal :param Patt: an instance of PatternFactory that has been primed (Patt.primer has been run) Note, primer sets the Amatrix, Fcalcs, energy, and flux :param refls: strong spots, we will simulate there vicinity and use the simultion overlap as a proxy for indexing solution quality :param data_image: the raw data image that is used to compute overlap """ detector = Patt.detector[Patt.panel_id] self.FS = detector.get_fast_axis() self.SS = detector.get_slow_axis() self.A = sqr(crystal.get_A()) self.U = sqr(crystal.get_U()) self.B = sqr(crystal.get_B()) self.s0 = Patt.beam.get_s0() self.ucell_abc = list(crystal.unit_cell().parameters())[:3] self.crystal = crystal self.Patt = Patt self.data = data_image self.rois = spot_utils.get_spot_roi( refls, dxtbx_image_size=detector.get_image_size(), szx=szx, szy=szy) self.spotX, self.spotY, _ = spot_utils.xyz_from_refl(refls) self.spot_mask = spot_utils.strong_spot_mask(refls, self.data.shape) self.counts = spot_utils.count_roi_overlap(self.rois, img_size=self.data.shape) if np.any(self.counts > 0): self.should_norm = True else: self.should_norm = False
Malls = {} for pid, img in izip(pids, pan_imgs): panel = detector[pid] rois = spot_utils.get_spot_roi(reflsPP[pid], dxtbx_image_size=panel.get_image_size(), szx=szx, szy=szy) counts = spot_utils.count_roi_overlap(rois, img_size=img.shape) roi_pp.append(rois) counts_pp.append(counts) spot_masks = spot_utils.strong_spot_mask(reflsPP[pid], img_sh, as_composite=False) # composite mask Mall = np.any(spot_masks, axis=0) yall, xall = np.where(Mall) # use comp mask to determine the boundary xlims[pid] = (xall.min() - szx, xall.max() + szx) ylims[pid] = (yall.max() + szy, yall.min() - szy) patches[pid] = get_spot_patches(spot_masks) Malls[pid] = spot_masks pan_img_idx = {pid: idx for idx, pid in enumerate(pids)}
def xyscan(crystal, fcalcs_energies, fcalcs, fracA, fracB, strong, rotxy_series, jid, mos_dom=1, mos_spread=0.05, flux=1e14, use_weights=False, raw_image=None): """ :param crystal: :param fcalcs_energies: :param fcalcs: :param fracA: :param fracB: :param strong: :param rotxy_series: :param jid: :param mos_dom: :param mos_spread: :param flux: :param use_weights: :return: """ Patts = sim_utils.PatternFactory() Patts.adjust_mosaicity(mos_dom, mos_spread) # defaults flux_per_en = [fracA * flux, fracB * flux] img_size = Patts.detector.to_dict()['panels'][0]['image_size'] found_spot_mask = spot_utils.strong_spot_mask(strong, img_size) #if use_weights: # if raw_image is None: # raise ValueError("Cant use weights if raw image is None") # spot_signals = raw_image * found_spot_mask # weights = spot_signals / spot_signals.max() overlaps = [] for rots in rotxy_series: if len(rots) == 2: Op = rots[0] * rots[1] elif len(rots) == 3: Op = rots[0] * rots[1] * rots[2] elif len(rots == 1): Op = rots[0] else: raise ValueError("rotxy_series should be list of 1,2, or 3-tuples") sim_patt = Patts.make_pattern2(crystal=deepcopy(crystal), flux_per_en=flux_per_en, energies_eV=fcalcs_energies, fcalcs_at_energies=fcalcs, mosaic_spread=None, mosaic_domains=None, ret_sum=True, Op=Op) if use_weights: dblock = utils.datablock_from_numpyarrays(image=sim_patt, detector=Patts.detector, beam=Patts.beam) sim_refl = flex.reflection_table.from_observations( dblock, params=find_spot_params) sim_spot_mask = spot_utils.strong_spot_mask( sim_refl, sim_patt.shape) overlaps.append(np.sum(sim_spot_mask * found_spot_mask)) #weights2 = sim_patt / sim_patt.max() #overlaps.append( np.sum(sim_sig_mask * found_spot_mask * weights * weights2)) else: sim_sig_mask = sim_patt > 0 overlaps.append(np.sum(sim_sig_mask * found_spot_mask)) print "JOB %d" % jid return overlaps
def run_paramList(Ntrials, odir, tag, rank, n_jobs, pkl_file): import os import sys from copy import deepcopy import numpy as np import h5py from IPython import embed from scipy.ndimage.morphology import binary_dilation import scitbx from scitbx.array_family import flex from scitbx.matrix import sqr,col from simtbx.nanoBragg import shapetype from simtbx.nanoBragg import nanoBragg import libtbx.load_env # possibly implicit from libtbx.development.timers import Profiler from dxtbx.model.crystal import CrystalFactory from cctbx import crystal,crystal_orientation from cxid9114 import utils from cxid9114.sim import sim_utils from cxid9114.spots import spot_utils from cxid9114.bigsim.bigsim_geom import DET,BEAM from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW from cxid9114.refine.jitter_refine import make_param_list from cxid9114.bigsim import sim_spectra from LS49.sim.step4_pad import microcrystal data_pack = utils.open_flex(pkl_file) CRYST = data_pack['crystalAB'] mos_spread_deg=0.015 mos_doms=1000 beam_size_mm=0.001 exposure_s=1 use_microcrystal=True Deff_A = 2200 length_um = 2.2 timelog = False crystal = microcrystal(Deff_A = Deff_A, length_um = length_um, beam_diameter_um = beam_size_mm*1000, verbose=False) spec_file = h5py.File("simMe_data_run62.h5", "r") spec_data = spec_file["hist_spec"] Umat_data = spec_file["Umats"] en_chans = spec_file["energy_bins"][()] ilow = np.abs(en_chans - ENERGY_LOW).argmin() ihigh = np.abs(en_chans - ENERGY_HIGH).argmin() wave_chans = ENERGY_CONV/en_chans sfall_main = sim_spectra.load_spectra("test_sfall.h5") refls_strong = data_pack['refls_strong'] strong_mask_img = spot_utils.strong_spot_mask( refls_strong, (1800,1800) ) # edge detection in the ground truth strong mask image reference_img = (binary_dilation(strong_mask_img, iterations=1).astype(int) - strong_mask_img.astype(int) ).astype(bool) param_fileout = os.path.join( odir, "rank%d_%s.pkl" % (rank, tag)) param_list = make_param_list( CRYST, DET, BEAM, Ntrials, rot=0.09, cell=0.1, eq=(1,1,0), min_Ncell=20, max_Ncell=40, min_mos_spread=0.005, max_mos_spread=0.02) for p in param_list: print(p['crystal'].get_unit_cell().parameters()) shot_idx = int(data_pack['img_f'].split("_")[-1].split(".")[0]) Fluxes = spec_data[2] #shot_idx] Pmax = param_list[0] F1max = 0 for i_trial in range(Ntrials): print ("<><><><><><><><><><><><><><>") print ("Job %d; Trial %d / %d" % (rank, i_trial+1, Ntrials)) print ("<><><><><><><><><><><><><><>") if (rank==0 and i_trial % smi_stride==0): print("GPU status") os.system("nvidia-smi") print("\n\n") print("CPU memory usage") mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'""" os.system(mem_usg) assert (len(wave_chans)==len(Fluxes)==len(sfall_main)) if np.sum(Fluxes)==0: print ("Cannot simulate with an all-zeros spectrum!") sys.exit() N = crystal.number_of_cells(sfall_main[0].unit_cell()) Ncells_abc = (N,N,N) if force_twocolor: Fluxes *= 0 Fluxes[ilow] = 1e12 Fluxes[ihigh]=1e12 P = param_list[i_trial] simsAB = sim_utils.sim_twocolors2( P['crystal'], DET, BEAM, sfall_main, en_chans, Fluxes, pids = None, profile="gauss", oversample=0, Ncells_abc = Ncells_abc, #P['Ncells_abc'], mos_dom=mos_doms, verbose=verbose, mos_spread=mos_spread_deg, #@mos_spread=P['mos_spread'], cuda=True, device_Id=rank, beamsize_mm=beamsize_mm, exposure_s=exposure_s, boost=crystal.domains_per_crystal) out = np.sum( [ simsAB[i][0] for i in simsAB.keys() if simsAB[i]], axis=0) if out.shape==(): print("This simsAB output has an empty shape, something is wrong!") sys.exit() trial_refls = spot_utils.refls_from_sims([out], DET, BEAM, thresh=thresh) trial_spotmask = spot_utils.strong_spot_mask( trial_refls, (1800,1800) ) trial_img = (binary_dilation(trial_spotmask, iterations=1).astype(int) - trial_spotmask.astype(int) ).astype(bool) comp_img = trial_img.astype(int) + reference_img.astype(int)*2 Nfalse_neg = (comp_img==2).sum() # reference has signal but trial doesnt Nfalse_pos = (comp_img==1).sum() # trial has signal but reference doesnt Ntrue_pos = (comp_img==3).sum() # Precision = float(Ntrue_pos) / (Ntrue_pos + Nfalse_pos) Recall = float(Ntrue_pos) / (Ntrue_pos + Nfalse_neg) F1 = 2.*(Precision*Recall) / (Precision+Recall) if F1 > F1max: Pmax = {'crystal': P['crystal'], 'mos_spread': P['mos_spread'], 'Ncells_abc': P['Ncells_abc'], "F1": F1} F1max = F1 print("Rank %d, Trial %d: F1score = %.5f" % (rank, i_trial, F1)) utils.save_flex(Pmax, param_fileout) return F1max
dump_file = "%s/results/run%d/dump_%d_feb8th.pkl" % (root, run, idx) f = h5py.File( "%s/videos/run%d/shot%d%s/dump_%d_feb8th_jitt.h5py" % (root, run, idx, tag, idx), "r") pids = f['pids'].value #pan_imgs = f['pan_imgs'].value # for the simulated images keys = [k for k in f.keys() if k.startswith("sim_imgs_")] Ntrials = len(keys) d = utils.open_flex(dump_file) R = d['refls_strong'] Rpp = spot_utils.refls_by_panelname(R) Masks = spot_utils.strong_spot_mask(R, (185, 194), as_composite=False) min_prob = 0.1 / Ntrials Ntotal_bad = Ntotal_spots = 0 for pidx, pid in enumerate(pids): sim = np.array([f[k][pidx] for k in keys]) im = sim.mean(0) im /= im.max() # make it a probability M0 = spot_utils.strong_spot_mask(Rpp[pid], (185, 194), as_composite=False) vals = [] for m in M0: vec = m * im val = vec[vec > 0].mean()
def integrate2(R, badmask, data, gain=28, fit_bg=True, zscore=8, sz=8): """ get the crystal scatter, background scatter, and photon counting noise for the reflections listed in the table R :param R: reflection table :param badmask: mask in numpy format, same shape as data :param data: data :param gain: detector gain per photon :return: 3 arrays, one for signal, background and noise """ from dials.algorithms.shoebox import MaskCode fg_code = MaskCode.Foreground.real Nrefl = len(R) fs_dim = 194 ss_dim = 185 Rpp = spot_utils.refls_by_panelname( R ) # this is a dictionary whose key (0-63) unlock that panels reflections allspotmask = {} for pid in Rpp: # load the spot mask for all strong spots for this panel allspotmask[pid] = spot_utils.strong_spot_mask(Rpp[pid], (ss_dim, fs_dim)) signa = np.zeros(Nrefl) bg = np.zeros(Nrefl) noise = np.zeros_like(signa) pix_per = np.zeros(Nrefl, int) for i_r, refl in enumerate(R): pid = refl['panel'] spotmask = refl['shoebox'].mask.as_numpy_array() & fg_code == fg_code f1, f2, s1, s2, _, _ = refl[ 'shoebox'].bbox # fast scan and slow scan edges of bounding box icent, jcent, _ = refl['xyzobs.px.value'] thisspotmask = np.zeros_like(allspotmask[pid]) thisspotmask[s1:s2, f1:f2] = spotmask i1 = int(max(icent - .5 - sz, 0)) i2 = int(min(icent - .5 + sz, fs_dim)) j1 = int(max(jcent - .5 - sz, 0)) j2 = int(min(jcent - .5 + sz, ss_dim)) sub_data = data[pid][j1:j2, i1:i2] sub_mask = ((~allspotmask[pid]) * badmask[pid])[j1:j2, i1:i2] sub_thisspotmask = thisspotmask[j1:j2, i1:i2] Is = sub_data[sub_thisspotmask].sum() if fit_bg: tilt, bgmask, coeff = tilting_plane(sub_data, mask=sub_mask, zscore=zscore) bg_fit_mask = np.logical_and(~bgmask, sub_mask) m = sub_thisspotmask.sum() n = bg_fit_mask.sum() m2n = float(m) / float( n ) # ratio of number of background to number of strong spot pixels # modifuf Is according to background plane fit Is = Is - tilt[sub_thisspotmask].sum() # store background pix according to Leslie 99 Ibg = m2n * sub_data[bg_fit_mask].sum() else: Ibg = 0 signa[i_r] = Is # signal in the spot bg[i_r] = Ibg # background around the spot noise[i_r] = (Is + Ibg + m2n * Ibg) / gain pix_per[i_r] = thisspotmask.sum() return signa, bg, noise, pix_per
os.path.join(indir, f) for f in os.listdir(indir) if f.endswith(".pkl") ][0] print h5_name, pkl_name f = h5py.File(h5_name, "r") D = utils.open_flex(pkl_name) print "Using %d reflections!" % len(D) Dpp = spot_utils.refls_by_panelname(D) pids = f['pids'].value pid_map = {p: i for i, p in enumerate(pids)} Nsim = sum([1 for k in f.keys() if k.startswith('sim_imgs')]) dsets = [f['sim_imgs_%d' % x] for x in range(Nsim)] Dpp_pids = Dpp.keys() strong_mask = array( [spot_utils.strong_spot_mask(Dpp[p], img_sh) for p in Dpp_pids]) thresh = 2 # 2 photon threshold scores = {} for i, mask in enumerate(strong_mask): pid = Dpp_pids[i] dset_idx = pid_map[pid] pan = array([d[dset_idx] > thresh for d in dsets]) scores[pid] = [(~logical_xor(mask, p)).sum() for p in pan] vals = [] for p in Dpp_pids: s = scores[p] mx = max(s) idx = where(array(s) == mx)[0] vals += list(idx)