def tst_all(quick=False, prefix="step5"):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
#
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
if quick: prefix_root = prefix + "_%06d"
else: prefix_root = prefix + "laue_%06d"
Nimages = 1 # 10000
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(Nimages)
for iteration in range(Nimages):
file_prefix = prefix_root % iteration
rand_ori = sqr(random_orientations[iteration])
run_sim2smv(prefix=file_prefix,
crystal=C,
spectra=iterator,
rotation=rand_ori,
quick=quick,
rank=0)
def tst_all(quick=False, prefix="step6"):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
#
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
mt = flex.mersenne_twister(seed=0)
if quick: prefix_root = prefix + "_%06d"
else: prefix_root = prefix + "poly_%06d"
Nimages = 1 # 10000
for iteration in range(Nimages):
file_prefix = prefix_root % iteration
rand_ori = sqr(mt.random_double_r3_rotation_matrix())
run_sim2smv(prefix=file_prefix,
crystal=C,
spectra=iterator,
rotation=rand_ori,
quick=quick,
rank=0)
def sim_all(Nshot_max=None, jid=0, n_jobs=1):
odir = "/global/project/projectdirs/lcls/dermen/bigsim"
prefix = odir + "/run62_%06d"
spec_file = h5py.File(full_path("simMe_data_run62.h5"), "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
wave_chans = ENERGY_CONV / en_chans
sfall = sim_spectra.load_spectra(full_path("test_sfall.h5"))
# his jobs id
#jid = int(sys.argv[1])
#global device_Id
device_Id = jid
Nshot = spec_data.shape[0]
if Nshot_max is not None:
Nshot = min(Nshot_max, Nshot)
print("Job %d: Simulating %d shots" % (jid, Nshot))
# total number of available jobs
#n_jobs = int(sys.argv[2])
idx_range = np.array_split(np.arange(Nshot), n_jobs)
Nshot_per_job = len(idx_range[jid])
for idx in idx_range[jid]:
print("<><><><><><>")
print("Job %d; Image %d / %d" % (jid, idx + 1, Nshot_per_job))
print("<><><><><><>")
if (jid == 0):
print("GPU status")
os.system("nvidia-smi")
spec = spec_data[idx]
C_ori = sqr(Umat_data[idx])
wavelength_A = np.mean(wave_chans)
C = microcrystal(Deff_A=Deff_A,
length_um=length_um,
beam_diameter_um=beam_size_mm * 1000,
verbose=False)
iterator = iter([(wave_chans, spec, wavelength_A)])
run_sim2smv(
prefix=prefix % idx, # "run62_%06d" % idx,
crystal=C,
spectra=iterator,
sfall_main=sfall,
rotation=C_ori,
quick=False,
rank=jid,
save_bragg=False)
def tst_one(quick=False, prefix="step5"):
spec_file = h5py.File(full_path("test_data.h5"), "r")
idx = 2
en_chans = spec_file["energy_bins"][()]
wave_chans = ENERGY_CONV / en_chans
spec = spec_file["hist_spec"][idx]
C_ori = sqr(spec_file["Umats"][idx])
wavelength_A = np.mean(wave_chans)
sfall = sim_spectra.load_spectra(full_path("test_sfall.h5"))
C = microcrystal(Deff_A=Deff_A,
length_um=length_um,
beam_diameter_um=beam_size_mm * 1000)
iterator = iter([(wave_chans, spec, wavelength_A)])
run_sim2smv(prefix="tst_one_test_data_%06d" % idx,
crystal=C,
spectra=iterator,
sfall_main=sfall,
rotation=C_ori,
quick=False,
rank=0,
save_bragg=False) #, actually_use_smv=False)
print("## hello from rank %d of %d"%(rank,size),"with omp_threads=",omp_get_num_procs())
## assign jobs
rank_in_pdb, size_in_pdb, ranks_in_pdb = jobAssign(size=size, num_pdb=simparams.num_pdbs, num_img=simparams.num_img[0])
import datetime
start_elapse = time.time()
if rank == 0:
print("Rank 0 time", datetime.datetime.now())
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
C = microcrystal(Deff_A = simparams.Deff_A, length_um = simparams.length_um, beam_diameter_um = simparams.beam_diameter_um)
# assume smaller than 10 um crystals
mt = flex.mersenne_twister(seed=0)
random_orientations = []
for iteration in range( sum(simparams.num_img) ):
random_orientations.append( mt.random_double_r3_rotation_matrix() )
# for ii in range(10): print("## TOP 10 orientations = ", random_orientations[ii])
print("## total orientations = ", len(random_orientations))
transmitted_info = dict(spectra = SS, crystal = C, random_orientations = random_orientations)
for idx_pdb in range( len(simparams.pdb_files) ):
save_folder = "./" + simparams.prefix + "_" + str(idx_pdb).zfill(3)
if not os.path.isdir(save_folder):
print("## creating folder: ", save_folder)
iterator = spectra.generate_recast_renormalized_image(image=image,energy=7150.,total_flux=1e12)
quick = False
if quick: prefix_root="step4Y_%06d"
else: prefix_root="step4B_%06d"
file_prefix = prefix_root%image
rand_ori = sqr(random_orientation)
from LS49.sim.step4_pad import run_sim2smv
run_sim2smv(prefix = file_prefix,crystal = crystal,spectra=iterator,rotation=rand_ori,quick=quick)
if __name__=="__main__":
from LS49.spectra.generate_spectra import spectra_simulation
from LS49.sim.step4_pad import microcrystal
SS = spectra_simulation()
C = microcrystal(Deff_A = 4000, length_um = 1., beam_diameter_um = 1.0) # assume smaller than 10 um crystals
mt = flex.mersenne_twister(seed=0)
random_orientations = []
N_total = 20160
N_stride = 70 # total number of jobs
for iteration in range(N_total):
random_orientations.append( mt.random_double_r3_rotation_matrix() )
import sys
job_no = int(sys.argv[1])
for idx in range(job_no,N_total,N_stride):
print(("idx------------------->",idx))
tst_one(image=idx,spectra=SS,crystal=C,random_orientation=random_orientations[idx])
print("OK")
"""bsub -n 8 -o job${JOB}.log OMP_NUM_THREADS=8 libtbx.python ~/proj-1217/modules/LS49/sim/step4batch_pad.py ${JOB}
for JOB in `seq 10 69`; do bsub -q psanaq -n 8 -o job${JOB}.log OMP_NUM_THREADS=8 libtbx.python ~/proj-1217/modules/LS49/sim/step4batch_pad.py ${JOB};done
def run_sim2smv(Nshot_max, odir, prefix, rank, n_jobs, save_bragg=False,
save_smv=True, save_h5 =False, return_pixels=False):
from six.moves import range, StringIO
from six.moves import cPickle as pickle
from cxid9114.sim import sim_utils
import os
import h5py
import math
import sys
import numpy as np
from IPython import embed
from cxid9114.bigsim.bigsim_geom import DET,BEAM
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from cctbx import crystal,crystal_orientation
from LS49.sim.step4_pad import microcrystal
from cxid9114 import utils
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.bigsim import sim_spectra
odir_j = os.path.join( odir, "job%d" % rank)
if not os.path.exists(odir_j):
os.makedirs(odir_j)
add_noise = False
add_background = False
overwrite = True #$False
sample_thick_mm = 0.005 # 50 micron GDVN nozzle makes a ~5ish micron jet
air_thick_mm =0 # mostly vacuum, maybe helium layer of 1 micron
flux_ave=2e11
add_spots_algorithm="cuda"
big_data = "." # directory location for reference files
detpixels_slowfast = (1800,1800)
pixsize_mm=0.11
distance_mm = 125
offset_adu=30
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True #False
Ncells_abc=(120,120,120)
Deff_A = 2200
length_um = 2.2
timelog = False
background = utils.open_flex("background")
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
Nshot = spec_data.shape[0]
idx_range = np.array_split(np.arange(Nshot), n_jobs)
Nshot_per_job = len(idx_range[rank])
if Nshot_max > 0 :
Nshot_per_job = min( Nshot_max, Nshot_per_job)
print ("Job %d: Simulating %d shots" % (rank, Nshot_per_job))
istart = idx_range[rank][0]
istop = istart + Nshot_per_job
smi_stride = 10
for idx in range( istart, istop):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Image %d (%d - %d)" % (rank, idx+1, istart, istop))
print ("<><><><><><><><><><><><><><>")
smv_fileout = os.path.join( odir_j, prefix % idx + ".img")
h5_fileout = smv_fileout + ".h5"
if os.path.exists(smv_fileout) and not overwrite and save_smv:
print("Shot %s exists: moving on" % smv_fileout)
continue
if os.path.exists(h5_fileout) and not overwrite and save_h5:
print("Shot %s exists: moving on" % h5_fileout)
continue
if (rank==0 and idx % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
spec = spec_data[2]
rotation = sqr(Umat_data[2])
wavelength_A = np.mean(wave_chans)
spectra = iter([(wave_chans, spec, wavelength_A)])
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
assert (len(wavlen)==len(flux)==len(sfall_main))
N = crystal.number_of_cells(sfall_main[0].unit_cell())
if use_mcrocrystal:
Ncells_abc = (N,N,N)
flux *= 0
flux[ilow] = 1e12
flux[ihigh]=1e12
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mos_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(mos_doms)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
Amatrix_rot = (rotation * sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose()
#SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast,
# pixel_size_mm=pixsize_mm,Ncells_abc=Ncells_abc,
# wavelength_A=wavelength_A,verbose=verbose)
SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose)
SIM.adc_offset_adu = offset_adu # Do not offset by 40
SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = mos_doms
SIM.distance_mm=distance_mm
SIM.set_mosaic_blocks(UMAT_nm)
SIM.seed = 1
SIM.oversample=1
SIM.wavelength_A = wavelength_A
SIM.polarization=1
SIM.default_F=0
SIM.Fhkl=sfall_main[0].amplitudes()
SIM.progress_meter=False
SIM.flux=flux_ave
SIM.exposure_s=exposure_s
SIM.beamsize_mm=beam_size_mm
SIM.Ncells_abc=Ncells_abc
SIM.xtal_shape=shapetype.Gauss
idxpath = "try3_idx2/job0/dump_0_data.pkl"
Amat = sim_utils.Amatrix_dials2nanoBragg(utils.open_flex(idxpath)["crystalAB"])
#SIM.Umatrix = rotation.elems
#SIM.unit_cell_Adeg = sfall_main[0].unit_cell()
#SIM2 = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose)
#SIM2.adc_offset_adu = offset_adu # Do not offset by 40
#SIM2.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
#SIM2.mosaic_domains = mos_doms
#SIM2.distance_mm=distance_mm
#SIM2.set_mosaic_blocks(UMAT_nm)
#SIM2.seed = 1
#SIM2.oversample=1
#SIM2.wavelength_A = wavelength_A
#SIM2.polarization=1
#SIM2.default_F=0
#SIM2.Fhkl=sfall_main[0].amplitudes()
#SIM2.progress_meter=False
#SIM2.flux=flux_ave
#SIM2.exposure_s=exposure_s
#SIM2.beamsize_mm=beam_size_mm
#SIM2.Ncells_abc=Ncells_abc
#SIM2.xtal_shape=shapetype.Gauss
#SIM2.Umatrix = rotation.elems
#SIM2.unit_cell_Adeg = sfall_main[0].unit_cell()
#SIM.Amatrix_RUB = Amatrix_rot
#SIM.Amatrix = Amatrix_rot.inverse()
raw_pixel_sum = flex.double(len(SIM.raw_pixels))
Nflux = len(flux)
print("Beginning the loop")
for x in range(Nflux):
if x % 10==0:
print("+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" % (x+1, Nflux) , end="\r")
if flux[x] ==0:
continue
#SIM.Amatrix = Amatrix_rot.inverse()
print (SIM.Ncells_abc)
SIM.wavelength_A=wavlen[x]
SIM.flux=flux[x]
SIM.Fhkl=sfall_main[x].amplitudes()
SIM.Amatrix = Amat#Amatrix_rot.inverse()
#sim_utils.compare_sims(SIM, SIM2)
#SIM.Ncells_abc=Ncells_abc
#SIM.adc_offset_adu = offset_adu
#SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
#SIM.mosaic_domains = mos_doms #
#SIM.distance_mm=distance_mm
#SIM.set_mosaic_blocks(UMAT_nm)
#SIM.seed = 1
#SIM.polarization=1
#SIM.default_F=0
#SIM.xtal_shape=shapetype.Gauss
#SIM.progress_meter=False
#SIM.exposure_s = exposure_s
#SIM.beamsize_mm=beam_size_mm
SIM.timelog=timelog
SIM.device_Id=rank
SIM.raw_pixels *= 0 # just in case!
SIM.add_nanoBragg_spots_cuda()
if use_microcrystal:
raw_pixel_sum += SIM.raw_pixels * crystal.domains_per_crystal
print()
SIM.raw_pixels = raw_pixel_sum
if add_background:
SIM.raw_pixels = SIM.raw_pixels + background
SIM.detector_psf_kernel_radius_pixels=5;
#SIM.detector_psf_fwhm_mm=0.08;
#SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian)
SIM.detector_psf_type=shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm=0
SIM.quantum_gain = 28.
#SIM.apply_psf()
if add_noise:
SIM.add_noise() #converts phtons to ADU.
extra = "PREFIX=%s;\nRANK=%d;\n"%(prefix,rank)
out = SIM.raw_pixels.as_numpy_array()
if save_smv:
SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True)
elif save_h5:
f = h5py.File(h5_fileout, "w")
f.create_dataset("bigsim_d9114",
data=SIM.raw_pixels.as_numpy_array().astype(np.uint16).reshape(detpixels_slowfast),
compression="lzf")
f.close()
if npout is not None:
np.save(npout, SIM.raw_pixels.as_numpy_array())
SIM.free_all()
cryst_name = "cryst0"
#cryst_name = "ps2.crystR.pkl"
output_basename = cryst_name.replace(".pkl", tag)
# load the project beam, crystal-base model, and detector
cryst = utils.open_flex(cryst_name)
###
# THIS SNIPPET WILL BOOST THE XTAL SCATTER
###
from LS49.sim.step4_pad import microcrystal
ucell = cryst.get_unit_cell()
a, b, c, _, _, _ = ucell.parameters()
Na, Nb, Nc = Ncells_abc
size = np.power(a * Na * b * Nb * c * Nc, 1 / 3.)
microC = microcrystal(Deff_A=size, length_um=4, beam_diameter_um=4)
Iboost = microC.domains_per_crystal
####
print("Doing the starting sim")
# generate base crystal pattern and
t = time.time()
simsAB, Patt = sim_utils.sim_twocolors2(cryst,
det,
beam,
fcalcs=FF,
energies=ENERGIES,
fluxes=FLUX,
pids=panel_ids,
profile='gauss',
oversample=0,
def run_paramList(Ntrials, odir, tag, rank, n_jobs, pkl_file):
import os
import sys
from copy import deepcopy
import numpy as np
import h5py
from IPython import embed
from scipy.ndimage.morphology import binary_dilation
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from dxtbx.model.crystal import CrystalFactory
from cctbx import crystal,crystal_orientation
from cxid9114 import utils
from cxid9114.sim import sim_utils
from cxid9114.spots import spot_utils
from cxid9114.bigsim.bigsim_geom import DET,BEAM
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.refine.jitter_refine import make_param_list
from cxid9114.bigsim import sim_spectra
from LS49.sim.step4_pad import microcrystal
data_pack = utils.open_flex(pkl_file)
CRYST = data_pack['crystalAB']
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True
Deff_A = 2200
length_um = 2.2
timelog = False
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
refls_strong = data_pack['refls_strong']
strong_mask_img = spot_utils.strong_spot_mask(
refls_strong, (1800,1800) )
# edge detection in the ground truth strong mask image
reference_img = (binary_dilation(strong_mask_img, iterations=1).astype(int) -
strong_mask_img.astype(int) ).astype(bool)
param_fileout = os.path.join( odir, "rank%d_%s.pkl" % (rank, tag))
param_list = make_param_list(
CRYST, DET, BEAM, Ntrials,
rot=0.09, cell=0.1, eq=(1,1,0),
min_Ncell=20, max_Ncell=40,
min_mos_spread=0.005, max_mos_spread=0.02)
for p in param_list:
print(p['crystal'].get_unit_cell().parameters())
shot_idx = int(data_pack['img_f'].split("_")[-1].split(".")[0])
Fluxes = spec_data[2] #shot_idx]
Pmax = param_list[0]
F1max = 0
for i_trial in range(Ntrials):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Trial %d / %d" % (rank, i_trial+1, Ntrials))
print ("<><><><><><><><><><><><><><>")
if (rank==0 and i_trial % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
assert (len(wave_chans)==len(Fluxes)==len(sfall_main))
if np.sum(Fluxes)==0:
print ("Cannot simulate with an all-zeros spectrum!")
sys.exit()
N = crystal.number_of_cells(sfall_main[0].unit_cell())
Ncells_abc = (N,N,N)
if force_twocolor:
Fluxes *= 0
Fluxes[ilow] = 1e12
Fluxes[ihigh]=1e12
P = param_list[i_trial]
simsAB = sim_utils.sim_twocolors2(
P['crystal'],
DET,
BEAM,
sfall_main,
en_chans,
Fluxes,
pids = None,
profile="gauss",
oversample=0,
Ncells_abc = Ncells_abc, #P['Ncells_abc'],
mos_dom=mos_doms,
verbose=verbose,
mos_spread=mos_spread_deg,
#@mos_spread=P['mos_spread'],
cuda=True,
device_Id=rank,
beamsize_mm=beamsize_mm,
exposure_s=exposure_s,
boost=crystal.domains_per_crystal)
out = np.sum( [ simsAB[i][0] for i in simsAB.keys() if simsAB[i]], axis=0)
if out.shape==():
print("This simsAB output has an empty shape, something is wrong!")
sys.exit()
trial_refls = spot_utils.refls_from_sims([out], DET, BEAM, thresh=thresh)
trial_spotmask = spot_utils.strong_spot_mask(
trial_refls, (1800,1800) )
trial_img = (binary_dilation(trial_spotmask, iterations=1).astype(int) -
trial_spotmask.astype(int) ).astype(bool)
comp_img = trial_img.astype(int) + reference_img.astype(int)*2
Nfalse_neg = (comp_img==2).sum() # reference has signal but trial doesnt
Nfalse_pos = (comp_img==1).sum() # trial has signal but reference doesnt
Ntrue_pos = (comp_img==3).sum() #
Precision = float(Ntrue_pos) / (Ntrue_pos + Nfalse_pos)
Recall = float(Ntrue_pos) / (Ntrue_pos + Nfalse_neg)
F1 = 2.*(Precision*Recall) / (Precision+Recall)
if F1 > F1max:
Pmax = {'crystal': P['crystal'],
'mos_spread': P['mos_spread'],
'Ncells_abc': P['Ncells_abc'], "F1": F1}
F1max = F1
print("Rank %d, Trial %d: F1score = %.5f" % (rank, i_trial, F1))
utils.save_flex(Pmax, param_fileout)
return F1max
def run_sim2smv(Nshot_max, odir, tag, rank, n_jobs, save_bragg=False,
save_smv=True, save_h5 =False, return_pixels=False):
import os
import h5py
import math
import sys
import numpy as np
from IPython import embed
from cxid9114.bigsim.bigsim_geom import DET,BEAM
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from cctbx import crystal,crystal_orientation
from LS49.sim.step4_pad import microcrystal
from cxid9114 import utils
from cxid9114.sim import sim_utils
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.bigsim import sim_spectra
from dxtbx.model.crystal import CrystalFactory
odir_j = os.path.join( odir, "job%d" % rank)
if not os.path.exists(odir_j):
os.makedirs(odir_j)
cryst_descr = {'__id__': 'crystal',
'real_space_a': (79, 0, 0),
'real_space_b': (0, 79, 0),
'real_space_c': (0, 0, 38),
'space_group_hall_symbol': '-P 4 2'}
Crystal = CrystalFactory.from_dict(cryst_descr)
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True
Deff_A = 2200
length_um = Deff_A/1000.
timelog = False
if add_background:
background = utils.open_flex("background").as_numpy_array()
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
Nshot = spec_data.shape[0]
idx_range = np.array_split(np.arange(Nshot), n_jobs)
Nshot_per_job = len(idx_range[rank])
if Nshot_max > 0 :
Nshot_per_job = min( Nshot_max, Nshot_per_job)
print ("Job %d: Simulating %d shots" % (rank, Nshot_per_job))
istart = idx_range[rank][0]
istop = istart + Nshot_per_job
smi_stride = 10
for idx in range( istart, istop):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Image %d (%d - %d)" % (rank, idx+1, istart, istop))
print ("<><><><><><><><><><><><><><>")
smv_fileout = os.path.join( odir_j, "%s_%d.img" % (tag,idx))
h5_fileout = smv_fileout + ".h5"
if os.path.exists(smv_fileout) and not overwrite and save_smv:
print("Shot %s exists: moving on" % smv_fileout)
continue
if os.path.exists(h5_fileout) and not overwrite and save_h5:
print("Shot %s exists: moving on" % h5_fileout)
continue
if (rank==0 and idx % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
if force_index is not None:
spec = spec_data[force_index]
rotation = sqr(Umat_data[force_index])
else:
spec = spec_data[idx]
rotation = sqr(Umat_data[idx])
wavelength_A = np.mean(wave_chans)
spectra = iter([(wave_chans, spec, wavelength_A)])
wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
assert (len(wavlen)==len(flux)==len(sfall_main))
if np.sum(flux)==0:
continue
N = crystal.number_of_cells(sfall_main[0].unit_cell())
Ncells_abc = (N,N,N)
if not on_axis:
Crystal.set_U(rotation)
if idx_path is not None:
assert( model_file is None)
Crystal = utils.open_flex(idx_path)['crystalAB']
if model_file is not None:
assert( idx_path is None)
P = utils.open_flex(model_file)
Crystal = P['crystal']
#mos_spread_deg = P['mos_spread']
#Ncells_abc = P['Ncells_abc']
if force_twocolor:
flux *= 0
flux[ilow] = 1e11
flux[ihigh]=1e11
simsAB = sim_utils.sim_twocolors2(
Crystal,
DET,
BEAM,
sfall_main,
en_chans,
flux,
pids = None,
profile="gauss",
oversample=1,
Ncells_abc = Ncells_abc,
mos_dom=mos_doms,
verbose=verbose,
mos_spread=mos_spread_deg,
cuda=True,
device_Id =rank,
beamsize_mm=beamsize_mm,
exposure_s=exposure_s,
accumulate=True,
boost=crystal.domains_per_crystal)
if not simsAB:
continue
out = simsAB[0]
if add_background:
out = out + background
if add_noise:
SIM = nanoBragg(detector=DET, beam=BEAM)
SIM.exposure_s = exposure_s
SIM.flux = np.sum(flux)
SIM.raw_pixels = flex.double(out.ravel())
SIM.detector_psf_kernel_radius_pixels=5;
SIM.detector_psf_type=shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm=0
SIM.quantum_gain = 28
SIM.add_noise()
out = SIM.raw_pixels.as_numpy_array().reshape(out.shape)
f = h5py.File(h5_fileout, "w")
f.create_dataset("bigsim_d9114",
data=out,
compression="lzf")
ua,ub,uc = Crystal.get_real_space_vectors()
f.create_dataset("real_space_a", data=ua)
f.create_dataset("real_space_b", data=ub)
f.create_dataset("real_space_c", data=uc)
f.create_dataset("space_group_hall_symbol",
data=Crystal.get_space_group().info().type().hall_symbol())
f.create_dataset("Umatrix", data=Crystal.get_U())
f.create_dataset("fluxes",data=flux)
f.create_dataset("energies", data=en_chans)
f.close()
if npout is not None:
np.save(npout, out) # SIM.raw_pixels.as_numpy_array())
def main(jid):
import sys
import glob
import os
from copy import deepcopy
import numpy as np
import pandas
from scipy.spatial import cKDTree, distance
from IPython import embed
import h5py
from cctbx import miller, sgtbx
from cxid9114 import utils
from dials.array_family import flex
import dxtbx
from cxid9114 import utils
from cxid9114.geom import geom_utils
from cxid9114.spots import integrate, spot_utils
from cxid9114 import parameters
from cxid9114.sim import sim_utils
from cxid9114.solvers import setup_inputs
from cxid9114.refine import metrics
from LS49.sim.step4_pad import microcrystal
assert (iglob is not None)
spec_f = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_f["hist_spec"][()]
sg96 = sgtbx.space_group(" P 4nw 2abw")
ofile = "%s_liftoff_betelgeuse%d.%d.pdpkl" % (tag, jid + 1, Njobs)
ofile = os.path.join(odir, ofile)
print(ofile)
file_list = glob.glob(iglob)
Nfiles = len(file_list)
ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH]
FF = [1e4, None]
FLUX = [1e12, 1e12]
beamsize_mm = 0.001
Deff_A = 2200
length_um = 2.2
detector = utils.open_flex("bigsim_detect.pkl")
beam = utils.open_flex("bigsim_beam.pkl")
beamA = deepcopy(beam)
beamB = deepcopy(beam)
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
beamA.set_wavelength(waveA)
beamB.set_wavelength(waveB)
file_list_idx = np.array_split(np.arange(Nfiles), Njobs)
crystal = microcrystal(Deff_A=Deff_A,
length_um=length_um,
beam_diameter_um=beamsize_mm * 1000,
verbose=False)
all_dfs = []
idxstart = file_list_idx[jid][0]
Nfiles = len(file_list_idx[jid])
if max_files is not None:
Nfiles = min(max_files, Nfiles)
for idx in range(idxstart, idxstart + Nfiles):
if jid == 0 and idx % smi_stride == 0:
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg = """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
data_name = file_list[idx]
data = utils.open_flex(data_name)
shot_idx = int(data["img_f"].split("_")[-1].split(".")[0])
print "Data file %s" % data_name
shot_idx = int(shot_idx)
shot_spectrum = spec_data[shot_idx]
chanA_flux = shot_spectrum[10:25].sum()
chanB_flux = shot_spectrum[100:115].sum()
crystalAB = data["crystalAB"]
print "Doing the basic simulation.."
simsAB = sim_utils.sim_twocolors2(
crystalAB,
detector,
beam,
FF, [parameters.ENERGY_LOW, parameters.ENERGY_HIGH],
FLUX,
Gauss=True,
oversample=0,
Ncells_abc=(25, 25, 25),
mos_dom=1000,
mos_spread=0.015,
cuda=cuda,
device_Id=jid,
beamsize_mm=beamsize_mm,
boost=crystal.domains_per_crystal,
exposure_s=1)
print "Done!"
refl_data = data["refls_strong"]
#
print "\n\n\n#######\nProcessing %d reflections read from the data file \n#####\n\n" % len(
refl_data)
refl_simA = spot_utils.refls_from_sims(simsAB[0],
detector,
beamA,
thresh=thresh)
refl_simB = spot_utils.refls_from_sims(simsAB[1],
detector,
beamB,
thresh=thresh)
residA = metrics.check_indexable2(refl_data, refl_simA, detector,
beamA, crystalAB, hkl_tol)
residB = metrics.check_indexable2(refl_data, refl_simB, detector,
beamB, crystalAB, hkl_tol)
print "Initial metrics suggest that:"
print "\t %d reflections could be indexed by channeL A" % residA[
'indexed'].sum()
print "\t %d reflections could be indexed by channeL B" % residB[
'indexed'].sum()
print "\t NOw we can check for outliers.. "
if plot_overlap:
spot_utils.plot_overlap(refl_simA, refl_simB, refl_data, detector)
d = {
"crystalAB": crystalAB,
"residA": residA,
"residB": residB,
"beamA": beamA,
"beamB": beamB,
"detector": detector,
"refls_simA": refl_simA,
"refls_simB": refl_simB,
"refls_data": refl_data
}
# integrate with tilt plane subtraction
print("LOADING THE FINE IMAGE")
loader = dxtbx.load(data["img_f"])
pan_data = np.array([loader.get_raw_data().as_numpy_array()])
print(data["img_f"])
# make a dummie mask
mask = np.ones_like(pan_data).astype(np.bool)
# before processing we need to check edge cases
print "Checking the simulations edge cases, basically to do with the spot detection of simulations... \n\t such a pain.. "
filt = True
if filt:
_, all_HiA, _ = spot_utils.refls_to_hkl(refl_simA,
detector,
beamA,
crystal=crystalAB,
returnQ=True)
all_treeA = cKDTree(all_HiA)
nnA = all_treeA.query_ball_point(all_HiA, r=1e-7)
_, all_HiB, _ = spot_utils.refls_to_hkl(refl_simB,
detector,
beamB,
crystal=crystalAB,
returnQ=True)
all_treeB = cKDTree(all_HiB)
nnB = all_treeB.query_ball_point(all_HiB, r=1e-7)
NreflA = len(refl_simA)
NreflB = len(refl_simB)
drop_meA = []
for i, vals in enumerate(nnA):
if i in drop_meA:
continue
if len(vals) > 1:
pids = [refl_simA[v]['panel'] for v in vals]
if len(set(pids)) == 1:
refl_vals = refl_simA.select(
flex.bool(
[i_v in vals for i_v in np.arange(NreflA)]))
x, y, z = spot_utils.xyz_from_refl(refl_vals)
allI = [r['intensity.sum.value'] for r in refl_vals]
allI = sum(allI)
xm = np.mean(x)
ym = np.mean(y)
zm = np.mean(z)
drop_meA.extend(vals[1:])
x1b, x2b, y1b, y2b, z1b, z2b = zip(
*[r['bbox'] for r in refl_vals])
keep_me = vals[0]
# indexing order is important to modify as reference
refl_simA['intensity.sum.value'][keep_me] = allI
refl_simA['xyzobs.px.value'][keep_me] = (xm, ym, zm)
refl_simA['bbox'][keep_me] = (min(x1b), max(x2b),\
min(y1b), max(y2b), min(z1b), max(z2b))
else:
drop_meA.append(vals)
print vals
if drop_meA:
keep_meA = np.array([i not in drop_meA for i in range(NreflA)])
refl_simA = refl_simA.select(flex.bool(keep_meA))
NreflA = len(refl_simA)
drop_meB = []
for i, vals in enumerate(nnB):
if i in drop_meB:
continue
if len(vals) > 1:
pids = [refl_simB[v]['panel'] for v in vals]
if len(set(pids)) == 1:
print vals
# merge_spots(vals)
refl_vals = refl_simB.select(
flex.bool(
[i_v in vals for i_v in np.arange(NreflB)]))
x, y, z = spot_utils.xyz_from_refl(refl_vals)
allI = [r['intensity.sum.value'] for r in refl_vals]
allI = sum(allI)
xm = np.mean(x)
ym = np.mean(y)
zm = np.mean(z)
drop_meB.extend(vals[1:])
x1b, x2b, y1b, y2b, z1b, z2b = zip(
*[r['bbox'] for r in refl_vals])
keep_me = vals[0]
refl_simB['intensity.sum.value'][keep_me] = allI
refl_simB['xyzobs.px.value'][keep_me] = (xm, ym, zm)
refl_simB['bbox'][keep_me] = (min(x1b), max(x2b), min(y1b),\
max(y2b), min(z1b), max(z2b))
else:
drop_meB.append(vals)
print vals
if drop_meB:
keep_meB = [i not in drop_meB for i in range(NreflB)]
refl_simB = refl_simB.select(flex.bool(keep_meB))
NreflB = len(refl_simB)
## remake the trees given the drops
_, all_HiA = spot_utils.refls_to_hkl(refl_simA,
detector,
beamA,
crystal=crystalAB,
returnQ=False)
all_treeA = cKDTree(all_HiA)
_, all_HiB = spot_utils.refls_to_hkl(refl_simB,
detector,
beamB,
crystal=crystalAB,
returnQ=False)
#all_treeB = cKDTree(all_HiB)
## CHECK if same HKL, indexed by both colors
# exists on multiple panels, and if so, delete...
nnAB = all_treeA.query_ball_point(all_HiB, r=1e-7)
drop_meA = []
drop_meB = []
for iB, iA_vals in enumerate(nnAB):
if len(iA_vals) > 0:
assert (len(iA_vals) == 1)
iA = iA_vals[0]
pidA = refl_simA[iA]['panel']
pidB = refl_simB[iB]['panel']
if pidA != pidB:
drop_meA.append(iA)
drop_meB.append(iB)
if drop_meA:
keep_meA = [i not in drop_meA for i in range(NreflA)]
refl_simA = refl_simA.select(flex.bool(keep_meA))
if drop_meB:
keep_meB = [i not in drop_meB for i in range(NreflB)]
refl_simB = refl_simB.select(flex.bool(keep_meB))
# ---- Done with edge case filters#
print "<><><><>\nI am doing checking the simulations for edge cases!\n<><><><>"
# reflections per panel
rpp = spot_utils.refls_by_panelname(refl_data)
rppA = spot_utils.refls_by_panelname(refl_simA)
rppB = spot_utils.refls_by_panelname(refl_simB)
DATA = {
"D": [],
"Dnoise": [],
"h": [],
"k": [],
"l": [],
"is_pos": [],
"hAnom": [],
"kAnom": [],
"lAnom": [],
"horig": [],
"korig": [],
"lorig": [],
"PA": [],
"PB": [],
"iA": [],
"iB": [],
"Nstrong": [],
"pid": [],
"delta_pix": []
} # NOTE: added in the delta pix
# for comparing sim and data center of masses
all_int_me = []
sz_fudge = sz = 5 # integration fudge factor to include spots that dont overlap perfectly with predictions
# double define for convenience cause sz is easier to type than sz_fudge
# now set up boundboxes and integrate
for idx_pid, pid in enumerate(rpp):
# NOTE: integrate the spots for this panel
Is, Ibk, noise, pix_per = integrate.integrate3(rpp[pid],
mask[pid],
pan_data[pid],
gain=nom_gain)
print "Processing peaks on CSPAD panel %d (%d / %d)" % (
pid, idx_pid, len(rpp))
R = rpp[pid]
if pid in rppA: # are there A-channel reflections on this panel
inA = True
RA = rppA[pid]
xA, yA, _ = spot_utils.xyz_from_refl(RA)
pointsA = np.array(zip(xA, yA))
HA, HiA, QA = spot_utils.refls_to_hkl(RA,
detector,
beamA,
crystal=crystalAB,
returnQ=True)
else:
inA = False
if pid in rppB: # are there B channel reflections on this channel
inB = True
RB = rppB[pid]
xB, yB, _ = spot_utils.xyz_from_refl(RB)
pointsB = np.array(zip(xB, yB))
HB, HiB, QB = spot_utils.refls_to_hkl(RB,
detector,
beamB,
crystal=crystalAB,
returnQ=True)
else:
inB = False
x, y, _ = spot_utils.xyz_from_refl(R)
x = np.array(x)
y = np.array(y)
panX, panY = detector[pid].get_image_size()
mergesA = []
mergesB = []
if inA and inB: # are there both A and B channel reflections ? If so, lets find out which ones have same hkl
# make tree structure for merging the spots
treeA = cKDTree(pointsA)
treeB = cKDTree(pointsB)
# how far apart should the two color spots be ?
# NOTE: this is the critical step - are the spots within rmax - and if so they are considered indexed..
rmax = geom_utils.twocolor_deltapix(detector[pid], beamA,
beamB)
merge_me = treeA.query_ball_tree(
treeB, r=rmax + sz_fudge) # slap on some fudge
# if pixels points in treeA are within rmax + sz_fugde of
# points in treeB, then these points are assumed to be overlapped
for iA, iB in enumerate(merge_me):
if not iB:
continue
iB = iB[0]
# check that the miller indices are the same
if not all([i == j for i, j in zip(HiA[iA], HiB[iB])]):
continue
x1A, x2A, y1A, y2A, _, _ = RA[iA]['bbox'] # shoebox'].bbox
x1B, x2B, y1B, y2B, _, _ = RB[iB]['bbox'] # shoebox'].bbox
xlow = max([0, min((x1A, x1B)) - sz])
xhigh = min([panX, max((x2A, x2B)) + sz])
ylow = max([0, min((y1A, y1B)) - sz])
yhigh = min([panY, max((y2A, y2B)) + sz])
# integrate me if I am in the bounding box!
int_me = np.where((xlow < x) & (x < xhigh) & (ylow < y)
& (y < yhigh))[0]
if not int_me.size:
continue
mergesA.append(iA)
mergesB.append(iB)
# integrate the spot, this will change depending on data or simulation
#NOTE : adding in the data-spot center of mass here as well
totalCOM = np.zeros(3) # NOTE: x,y,z
totalI = 0
totalNoise = 0
for ref_idx in int_me:
# TODO implement the spot intensity version here
# which fits the background plane!
totalI += Is[
ref_idx] #rpp[pid][ref_idx]["intensity.sum.value"]
totalNoise += noise[ref_idx]**2
totalCOM += np.array(
rpp[pid][ref_idx]["xyzobs.px.value"])
totalCOM /= len(int_me)
totalNoise = np.sqrt(totalNoise)
PA = RA[iA]['intensity.sum.value']
PB = RB[iB]['intensity.sum.value']
# NOTE: added the simulated spot(s) center of mass
posA = RA[iA]['xyzobs.px.value']
posB = RB[iB]['xyzobs.px.value']
simCOM = np.mean([posA, posB], axis=0)
# get the hkl structure factor, and the sym equiv hkl
(horig, korig,
lorig) = HiA[iA] # NOTE: same for A and B channels
h, k, l = setup_inputs.single_to_asu((horig, korig, lorig),
ano=False)
hAnom, kAnom, lAnom = setup_inputs.single_to_asu(
(horig, korig, lorig), ano=True)
if h == hAnom and k == kAnom and l == lAnom:
is_pos = True
else:
is_pos = False
DATA['is_pos'].append(is_pos)
DATA['horig'].append(horig)
DATA['korig'].append(korig)
DATA['lorig'].append(lorig)
DATA['h'].append(h)
DATA['k'].append(k)
DATA['l'].append(l)
DATA['hAnom'].append(hAnom)
DATA['kAnom'].append(kAnom)
DATA['lAnom'].append(lAnom)
DATA['D'].append(totalI)
DATA['Dnoise'].append(totalNoise)
DATA['PA'].append(PA)
DATA['PB'].append(PB)
DATA['pid'].append(pid)
DATA["Nstrong"].append(int_me.size)
DATA["iA"].append(iA)
DATA["iB"].append(iB)
all_int_me.append(int_me)
# NOTE: stash the sim-data distance (COM to COM)
DATA["delta_pix"].append(
distance.euclidean(totalCOM[:2], simCOM[:2]))
# this spot was both colors, overlapping
# find center of mass of all spots inside the integration box
# and find its distance to the center of mass of the simulation spots
if inA:
for iA, ref in enumerate(RA):
if iA in mergesA:
# this sim spot was already treated above
continue
x1A, x2A, y1A, y2A, _, _ = RA[iA][
'bbox'] # ['shoebox'].bbox
xlow = max((0, x1A - sz))
xhigh = min((panX, x2A + sz))
ylow = max((0, y1A - sz))
yhigh = min((panY, y2A + sz))
int_me = np.where((xlow < x) & (x < xhigh) & (ylow < y)
& (y < yhigh))[0]
if not int_me.size:
continue
# NOTE: added in the total sim calc
totalCOM = np.zeros(3)
totalI = 0
totalNoise = 0
for ref_idx in int_me:
# TODO implement the spot intensity version here
# which fits the background plane!
totalI += Is[
ref_idx] #rpp[pid][ref_idx]["intensity.sum.value"]
totalNoise += noise[ref_idx]**2
totalCOM += np.array(
rpp[pid][ref_idx]["xyzobs.px.value"])
totalCOM /= len(int_me)
totalNoise = np.sqrt(totalNoise)
PA = RA[iA]['intensity.sum.value']
PB = 0 # crucial ;)
# NOTE: added the simulated spot center of mass, for spotA
simCOM = np.array(RA[iA]['xyzobs.px.value'])
# get the hkl structure factor, and the sym equiv hkl
(horig, korig, lorig) = HiA[iA]
h, k, l = setup_inputs.single_to_asu((horig, korig, lorig),
ano=False)
hAnom, kAnom, lAnom = setup_inputs.single_to_asu(
(horig, korig, lorig), ano=True)
if h == hAnom and k == kAnom and l == lAnom:
is_pos = True
else:
is_pos = False
DATA['is_pos'].append(is_pos)
DATA['horig'].append(horig)
DATA['korig'].append(korig)
DATA['lorig'].append(lorig)
DATA['h'].append(h)
DATA['k'].append(k)
DATA['l'].append(l)
DATA['hAnom'].append(hAnom)
DATA['kAnom'].append(kAnom)
DATA['lAnom'].append(lAnom)
DATA['D'].append(totalI)
DATA['Dnoise'].append(totalNoise)
DATA['PA'].append(PA)
DATA['PB'].append(PB)
DATA['pid'].append(pid)
DATA["Nstrong"].append(int_me.size)
DATA["iA"].append(iA)
DATA["iB"].append(np.nan)
all_int_me.append(int_me)
# NOTE: stash the sim-data distance (COM to COM)
DATA["delta_pix"].append(
distance.euclidean(totalCOM[:2], simCOM[:2]))
if inB:
for iB, ref in enumerate(RB):
if iB in mergesB:
continue
x1B, x2B, y1B, y2B, _, _ = RB[iB]['bbox'] # shoebox'].bbox
xlow = max((0, x1B - sz))
xhigh = min((panX, x2B + sz))
ylow = max((0, y1B - sz))
yhigh = min((panY, y2B + sz))
# subimg = simsDataSum[pid][ylow:yhigh, xlow:xhigh]
# bg = 0
int_me = np.where((xlow < x) & (x < xhigh) & (ylow < y)
& (y < yhigh))[0]
if not int_me.size:
continue
# NOTE: added in the total COM calc
totalCOM = np.zeros(3)
totalI = 0
totalNoise = 0
for ref_idx in int_me:
# TODO implement the spot intensity version here
# which fits the background plane!
totalI += Is[
ref_idx] #rpp[pid][ref_idx]["intensity.sum.value"]
totalNoise += noise[ref_idx]**2
totalCOM += np.array(
rpp[pid][ref_idx]["xyzobs.px.value"])
totalCOM /= len(int_me)
totalNoise = np.sqrt(totalNoise)
PA = 0 # crucial ;)
PB = RB[iB]['intensity.sum.value']
# NOTE: added the simulated spot center of mass, for spotB only
simCOM = np.array(RB[iB]['xyzobs.px.value'])
# get the hkl structure factor, and the sym equiv hkl
(horig, korig, lorig) = HiB[iB]
h, k, l = setup_inputs.single_to_asu((horig, korig, lorig),
ano=False)
hAnom, kAnom, lAnom = setup_inputs.single_to_asu(
(horig, korig, lorig), ano=True)
if h == hAnom and k == kAnom and l == lAnom:
is_pos = True
else:
is_pos = False
DATA['is_pos'].append(is_pos)
DATA['horig'].append(horig)
DATA['korig'].append(korig)
DATA['lorig'].append(lorig)
DATA['h'].append(h)
DATA['k'].append(k)
DATA['l'].append(l)
DATA['hAnom'].append(hAnom)
DATA['kAnom'].append(kAnom)
DATA['lAnom'].append(lAnom)
DATA['D'].append(totalI)
DATA['Dnoise'].append(totalNoise)
DATA['PA'].append(PA)
DATA['PB'].append(PB)
DATA['pid'].append(pid)
DATA["Nstrong"].append(int_me.size)
DATA["iA"].append(np.nan)
DATA["iB"].append(iB)
all_int_me.append(int_me)
# NOTE: stash the sim-data distance (COM to COM)
DATA["delta_pix"].append(
distance.euclidean(totalCOM[:2], simCOM[:2]))
df = pandas.DataFrame(DATA)
df["run"] = run
df["shot_idx"] = shot_idx
df['LA'] = chanA_flux
df["LB"] = chanB_flux
df['K'] = FF[0]**2 * FLUX[0]
df['nominal_gain'] = nom_gain
all_dfs.append(df)
print("Saved %d partial structure factor measurements in file %s" %
(len(df), ofile))
DF = pandas.concat(all_dfs)
DF.to_pickle(ofile)
