def fine_xyrefine(refine_xy_file, n_jobs=6, raw_image=None, use_weights=False):
scan_data = utils.open_flex(refine_xy_file)
crystal = deepcopy(scan_data['optCrystal'])
refl = scan_data['refl']
x = col((1, 0, 0))
y = col((0, 1, 0))
xR = x.axis_and_angle_as_r3_rotation_matrix
yR = y.axis_and_angle_as_r3_rotation_matrix
fine_degs = np.arange(-0.045, 0.045, 0.005)
rotXY_series = [(xR(i, deg=True), yR(j, deg=True)) for i in fine_degs
for j in fine_degs]
fracA = int(scan_data["fracA"])
fracB = int(scan_data["fracB"])
fcalc_f = scan_data["fcalc_f"]
results = xyscan_multi(crystal,
fcalc_f,
fracA,
fracB,
refl,
rotXY_series,
n_jobs=n_jobs,
scan_func=xyscan,
use_weights=use_weights,
raw_image=raw_image)
optX_fine, optY_fine = rotXY_series[np.argmax(results)]
optA_fine = optX_fine * optY_fine * sqr(crystal.get_U()) * sqr(
crystal.get_B())
return results, optX_fine, optY_fine, optA_fine
def from_named_data(pkl_name):
from cxid9114 import utils
waveA = ENERGY_CONV / 8944.
IA = utils.open_flex(pkl_name) # this is refined miller data
out = IA.as_mtz_dataset(column_root_label="Iobs",
title="B",
wavelength=waveA)
out.add_miller_array(miller_array=IA.average_bijvoet_mates(),
column_root_label="IMEAN")
obj = out.mtz_object()
obj.write(pkl_name.replace(".pkl", ".mtz"))
def compress_masks(mask_pickles, outname):
"""
masks compress rather nicely
:param mask_pickles: mask pickle files containing dials masks
:param outname: hdf5 name
"""
with h5py.File(outname, "w") as h5:
for mask_f in mask_pickles:
mask = utils.open_flex(mask_f) # these are tuples in dials land
data = [m.as_numpy_array() for m in mask]
h5.create_dataset(mask_f,
data=data,
compression='gzip',
compression_opts=9,
shuffle=True)
def load_fcalc_for_default_spectrum():
if fcalc_file is None or not os.path.exists(fcalc_file):
n_jobs = effective_n_jobs()
wavelens_idx = np.array_split(range(len(wavelens_A)), n_jobs)
results = Parallel(n_jobs=n_jobs)(delayed(main)(wavelens_A, wavelens_idx[jid], jid)
for jid in range(n_jobs))
fcalc_at_wavelen = {}
for result_dict in results:
for k, v in result_dict.iteritems():
fcalc_at_wavelen.setdefault(k, []).append(v)
fcalc_at_wavelen[k] = fcalc_at_wavelen[k][0]
else:
fcalc_at_wavelen = utils.open_flex(fcalc_file)
return fcalc_at_wavelen
def xyscan_multi(crystal,
fcalcs_file,
fracA,
fracB,
strong,
scan_params,
n_jobs,
scan_func=xyscan):
"""
:param crystal_file:
:param energies:
:param fcalcs_file:
:param fracA:
:param fracB:
:param strong_file:
:param rotxy_series:
:param n_jobs:
:return:
"""
from joblib import Parallel, delayed
nscans = len(scan_params)
scan_idx = np.array_split(range(nscans), n_jobs)
scan_split = []
for idx in scan_idx:
scan_split.append([scan_params[i] for i in idx])
fcalcs_data = utils.open_flex(fcalcs_file)
results = Parallel(n_jobs=n_jobs)(delayed(scan_func)\
(crystal=crystal,
fcalcs_data=fcalcs_data,
fracA=fracA, fracB=fracB,
strong=strong,jid=jid,
rotxy_series=scan_split[jid]) \
for jid in range(n_jobs))
results = np.concatenate(results)
return results
import matplotlib as mpl
import time
import numpy as np
from cxid9114.refine.jitter_refine import JitterFactory
from cxid9114.sim import sim_utils
import pylab as plt
from cxid9114 import parameters
from cxid9114 import utils
from cxid9114.spots import spot_utils
import os
import sys
from dxtbx.model.detector import Detector
from scitbx.array_family import flex
# OPEN THE DETECTOR AND THE BEAM!
det = detector = utils.open_flex("xfel_det.pkl")
beam = utils.open_flex("beam0") #ps2.beam.pkl")
#beam = utils.open_flex("ps2.beam.pkl")
try:
tag = sys.argv[1]
outdir = tag
if not os.path.exists(outdir):
os.makedirs(outdir)
except IndexError:
tag = "" # DONT YOU EVEN CARE?
outdir = "OUTS/"
szx = szy = 11
spec_data = np.load("spec_gamma2.npz")
ENERGIES = spec_data['en'] - 2000
import cctbx
from cctbx import miller
from cctbx.crystal import symmetry
import os
import numpy as np
import six
import glob
from cxid9114 import utils
from cctbx.array_family import flex
from cctbx import sgtbx
from IPython import embed
symbol = args.symbol
ucell = args.unitcell
ftruth = utils.open_flex(args.truthpkl)
if six.PY3:
asu_map = np.load( os.path.join( args.datdir , "f_asu_map.npy"), allow_pickle=True )[()]
else:
asu_map = np.load( os.path.join( args.datdir , "f_asu_map.npy") )[()]
fcell_pos, asu_indices = zip(*asu_map.items())
flex_indices = flex.miller_index(asu_indices)
if args.trial is None:
data_files = glob.glob( os.path.join(args.datdir, "_fcell_iter*.npz") )
else:
data_files = glob.glob( os.path.join(args.datdir, "_fcell_trial%d_iter*.npz" % args.trial) )
# order files by iteration number
run = int(sys.argv[1])
nom_enA = int(sys.argv[2])
waveA_default = ENERGY_CONV / float(nom_enA)
fnames = glob.glob("results/run%d_alpha1alpha2/prepped*dumpsy2" % run)
offsets = {}
for pid in range(64):
offsets[pid] = []
all_f, all_iA, all_iB = [], [], []
waveBs, enBs, del_ens = [], [], []
del_spot_rad, del_spot_abs = [], []
resos = []
for f in fnames:
d = utils.open_flex(f)
rA = d['residA']
rB = d['residB']
beamA = d['beamA']
beamA.set_wavelength(waveA_default)
beamB = d['beamB']
det = d['detector']
refls = d['refls_data']
idxA = rA['indexed']
idxB = rB['indexed']
hkl = rA['hkl']
tree = KDTree(hkl)
pairs = tree.query_pairs(1e-3)
for i1, i2 in map(list, pairs):
if not idxA[i1] and not idxA[i2] and not idxB[i1] and not idxB[i2]:
continue
FLUX = [1e12, 1e12] # fluxes of the beams
flux_frac = np.random.uniform(.2, .8)
chanA_flux = flux_frac * 1e12
chanB_flux = (1. - flux_frac) * 1e12
FLUXdat = [chanA_flux, chanB_flux]
GAIN = np.random.uniform(0.5, 3)
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
exp_lst = ExperimentListFactory.from_json_file(
exp_name) # , check_format=False)
iset = exp_lst.imagesets()[0]
detector = iset.get_detector(0)
data = utils.open_flex(data_name)
beamA = deepcopy(iset.get_beam())
beamB = deepcopy(iset.get_beam())
beamA.set_wavelength(waveA)
beamB.set_wavelength(waveB)
crystalAB = data["crystalAB"]
if has_best:
print "Doing the optimized simulation, because you told me to"
szx = szy = 5
df = pandas.read_pickle(best_fname)
idxmax = df.ave_score.idxmax()
best_Amat = tuple(df.Amat.iloc[idxmax])
best_Ncell_abc = tuple(df[["Na", "Nb", "Nc"]].iloc[idxmax].values)
best_mos_spread = df["mos_spread"].iloc[idxmax]
degs = np.arange(-0.2, 0.2, 0.025)
fine_scan = False
use_weights = False
# ======================
# END OF PARAMETER ENTRY
# ======================
x = col((1., 0., 0.))
y = col((0., 1., 0.))
xRot = x.axis_and_angle_as_r3_rotation_matrix
yRot = y.axis_and_angle_as_r3_rotation_matrix
rotXY_series = [(xRot(i, deg=True), yRot(j, deg=True)) for i in degs
for j in degs]
loader = dxtbx.load("/Users/dermen/cxid9114/run62_hits_wtime.h5")
data = utils.open_flex(idx_proc_file)
hits_w_spec_ana = [
i for i in data
if data[i]['spectrum'] is not None and data[i]['can_analyze']
]
if Nprocess > -1:
hits_w_spec_ana = hits_w_spec_ana[:Nprocess]
for hit_i in hits_w_spec_ana:
crystal = data[hit_i]["crystals"][0]
refl = data[hit_i]['refl']
fracA = data[hit_i]['fracA']
fracB = data[hit_i]['fracB']
raw_img = loader.get_raw_data(hit_i).as_numpy_array()
#El = ExperimentListFactory.from_json_file(El_json, check_format=False)
iset = El.imagesets()[0]
fpath = iset.get_path(0)
h5 = h5py.File(fpath.replace(".npz", ""), 'r')
mos_spread = h5["mos_spread"][()]
Ncells_abc = tuple(h5["Ncells_abc"][()])
mos_doms = h5["mos_doms"][()]
profile = h5["profile"][()]
beamsize = h5["beamsize_mm"][()]
exposure_s = h5["exposure_s"][()]
spectrum = h5["spectrum"][()]
total_flux = np.sum(spectrum)
xtal_size = 0.0005 #h5["xtal_size_mm"][()]
refls_data = utils.open_flex(refl_pkl)
# TODO multi panel
# Make a strong spot mask that is used to fit tilting planes
FF = [1e3, None] # NOTE: not sure what to do here, we dont know the structure factor
FLUX = [total_flux * .5, total_flux*.5]
energies = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH]
if args.sad:
FF.pop()
FLUX.pop()
energies.pop()
BEAM = El.beams()[0]
DET = El.detectors()[0]
crystal = El.crystals()[0]
beams = []
from dials.algorithms.indexing.indexer import indexer_base
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from libtbx.phil import parse
from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color
import dxtbx
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
from dxtbx.model import Detector
import os
fcalc_f = "/Users/dermen/cxid9114_gain/sim/fcalc_slim.pkl"
outdir = "ssirp_res_det.refine"
if not os.path.exists(outdir):
os.makedirs(outdir)
BEAM = utils.open_flex(sim_utils.beam_f)
sad_wave = parameters.ENERGY_CONV / 8950
BEAM.set_wavelength(sad_wave)
MULTI_PANEL = True
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par_moder = deepcopy(spot_par)
spot_par.spotfinder.threshold.dispersion.global_threshold = 70.
spot_par.spotfinder.threshold.dispersion.gain = 28.
spot_par.spotfinder.threshold.dispersion.kernel_size = [4, 4]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 2.25
spot_par.spotfinder.threshold.dispersion.sigma_background = 6.
spot_par.spotfinder.filter.min_spot_size = 2
spot_par.spotfinder.force_2d = True
spot_par.spotfinder.lookup.mask = "../mask/dials_mask_64panels_2.pkl"
# coding: utf-8
from cxid9114 import utils
a=utils.open_flex('SA.pkl')
b=utils.open_flex('SB.pkl')
a
dir(a)
a.show_array()
b.show_array()
ba = b.show_array()
ba
b.show_array()
a.amplitudes()
a.amplitudes()[0]
dir(a)
get_ipython().magic(u'pinfo a.value_at_index')
a.value_at_index((1,9,8))
a.indices()
a.indices()[0]
H = [a.indices()[i] for i in len(a)]
H = [a.indices()[i] for i in len(a.indices())]
H = [i for i in a.indices()]
H
a.value_at_index(H[10])
b.value_at_index(H[10])
H[10]
a.value_at_index(H[1000])
b.value_at_index(H[1000])
a.resolution_range
get_ipython().magic(u'pinfo a.resolution_range')
def res_from_idx:
def res_from_idx:
def run_sim2smv(Nshot_max, odir, prefix, rank, n_jobs, save_bragg=False,
save_smv=True, save_h5 =False, return_pixels=False):
from six.moves import range, StringIO
from six.moves import cPickle as pickle
from cxid9114.sim import sim_utils
import os
import h5py
import math
import sys
import numpy as np
from IPython import embed
from cxid9114.bigsim.bigsim_geom import DET,BEAM
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from cctbx import crystal,crystal_orientation
from LS49.sim.step4_pad import microcrystal
from cxid9114 import utils
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.bigsim import sim_spectra
odir_j = os.path.join( odir, "job%d" % rank)
if not os.path.exists(odir_j):
os.makedirs(odir_j)
add_noise = False
add_background = False
overwrite = True #$False
sample_thick_mm = 0.005 # 50 micron GDVN nozzle makes a ~5ish micron jet
air_thick_mm =0 # mostly vacuum, maybe helium layer of 1 micron
flux_ave=2e11
add_spots_algorithm="cuda"
big_data = "." # directory location for reference files
detpixels_slowfast = (1800,1800)
pixsize_mm=0.11
distance_mm = 125
offset_adu=30
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True #False
Ncells_abc=(120,120,120)
Deff_A = 2200
length_um = 2.2
timelog = False
background = utils.open_flex("background")
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
Nshot = spec_data.shape[0]
idx_range = np.array_split(np.arange(Nshot), n_jobs)
Nshot_per_job = len(idx_range[rank])
if Nshot_max > 0 :
Nshot_per_job = min( Nshot_max, Nshot_per_job)
print ("Job %d: Simulating %d shots" % (rank, Nshot_per_job))
istart = idx_range[rank][0]
istop = istart + Nshot_per_job
smi_stride = 10
for idx in range( istart, istop):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Image %d (%d - %d)" % (rank, idx+1, istart, istop))
print ("<><><><><><><><><><><><><><>")
smv_fileout = os.path.join( odir_j, prefix % idx + ".img")
h5_fileout = smv_fileout + ".h5"
if os.path.exists(smv_fileout) and not overwrite and save_smv:
print("Shot %s exists: moving on" % smv_fileout)
continue
if os.path.exists(h5_fileout) and not overwrite and save_h5:
print("Shot %s exists: moving on" % h5_fileout)
continue
if (rank==0 and idx % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
spec = spec_data[2]
rotation = sqr(Umat_data[2])
wavelength_A = np.mean(wave_chans)
spectra = iter([(wave_chans, spec, wavelength_A)])
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
assert (len(wavlen)==len(flux)==len(sfall_main))
N = crystal.number_of_cells(sfall_main[0].unit_cell())
if use_mcrocrystal:
Ncells_abc = (N,N,N)
flux *= 0
flux[ilow] = 1e12
flux[ihigh]=1e12
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mos_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(mos_doms)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
Amatrix_rot = (rotation * sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose()
#SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast,
# pixel_size_mm=pixsize_mm,Ncells_abc=Ncells_abc,
# wavelength_A=wavelength_A,verbose=verbose)
SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose)
SIM.adc_offset_adu = offset_adu # Do not offset by 40
SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = mos_doms
SIM.distance_mm=distance_mm
SIM.set_mosaic_blocks(UMAT_nm)
SIM.seed = 1
SIM.oversample=1
SIM.wavelength_A = wavelength_A
SIM.polarization=1
SIM.default_F=0
SIM.Fhkl=sfall_main[0].amplitudes()
SIM.progress_meter=False
SIM.flux=flux_ave
SIM.exposure_s=exposure_s
SIM.beamsize_mm=beam_size_mm
SIM.Ncells_abc=Ncells_abc
SIM.xtal_shape=shapetype.Gauss
idxpath = "try3_idx2/job0/dump_0_data.pkl"
Amat = sim_utils.Amatrix_dials2nanoBragg(utils.open_flex(idxpath)["crystalAB"])
#SIM.Umatrix = rotation.elems
#SIM.unit_cell_Adeg = sfall_main[0].unit_cell()
#SIM2 = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose)
#SIM2.adc_offset_adu = offset_adu # Do not offset by 40
#SIM2.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
#SIM2.mosaic_domains = mos_doms
#SIM2.distance_mm=distance_mm
#SIM2.set_mosaic_blocks(UMAT_nm)
#SIM2.seed = 1
#SIM2.oversample=1
#SIM2.wavelength_A = wavelength_A
#SIM2.polarization=1
#SIM2.default_F=0
#SIM2.Fhkl=sfall_main[0].amplitudes()
#SIM2.progress_meter=False
#SIM2.flux=flux_ave
#SIM2.exposure_s=exposure_s
#SIM2.beamsize_mm=beam_size_mm
#SIM2.Ncells_abc=Ncells_abc
#SIM2.xtal_shape=shapetype.Gauss
#SIM2.Umatrix = rotation.elems
#SIM2.unit_cell_Adeg = sfall_main[0].unit_cell()
#SIM.Amatrix_RUB = Amatrix_rot
#SIM.Amatrix = Amatrix_rot.inverse()
raw_pixel_sum = flex.double(len(SIM.raw_pixels))
Nflux = len(flux)
print("Beginning the loop")
for x in range(Nflux):
if x % 10==0:
print("+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" % (x+1, Nflux) , end="\r")
if flux[x] ==0:
continue
#SIM.Amatrix = Amatrix_rot.inverse()
print (SIM.Ncells_abc)
SIM.wavelength_A=wavlen[x]
SIM.flux=flux[x]
SIM.Fhkl=sfall_main[x].amplitudes()
SIM.Amatrix = Amat#Amatrix_rot.inverse()
#sim_utils.compare_sims(SIM, SIM2)
#SIM.Ncells_abc=Ncells_abc
#SIM.adc_offset_adu = offset_adu
#SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
#SIM.mosaic_domains = mos_doms #
#SIM.distance_mm=distance_mm
#SIM.set_mosaic_blocks(UMAT_nm)
#SIM.seed = 1
#SIM.polarization=1
#SIM.default_F=0
#SIM.xtal_shape=shapetype.Gauss
#SIM.progress_meter=False
#SIM.exposure_s = exposure_s
#SIM.beamsize_mm=beam_size_mm
SIM.timelog=timelog
SIM.device_Id=rank
SIM.raw_pixels *= 0 # just in case!
SIM.add_nanoBragg_spots_cuda()
if use_microcrystal:
raw_pixel_sum += SIM.raw_pixels * crystal.domains_per_crystal
print()
SIM.raw_pixels = raw_pixel_sum
if add_background:
SIM.raw_pixels = SIM.raw_pixels + background
SIM.detector_psf_kernel_radius_pixels=5;
#SIM.detector_psf_fwhm_mm=0.08;
#SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian)
SIM.detector_psf_type=shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm=0
SIM.quantum_gain = 28.
#SIM.apply_psf()
if add_noise:
SIM.add_noise() #converts phtons to ADU.
extra = "PREFIX=%s;\nRANK=%d;\n"%(prefix,rank)
out = SIM.raw_pixels.as_numpy_array()
if save_smv:
SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True)
elif save_h5:
f = h5py.File(h5_fileout, "w")
f.create_dataset("bigsim_d9114",
data=SIM.raw_pixels.as_numpy_array().astype(np.uint16).reshape(detpixels_slowfast),
compression="lzf")
f.close()
if npout is not None:
np.save(npout, SIM.raw_pixels.as_numpy_array())
SIM.free_all()
def load_fcalc_file(fcalc_file):
fcalcs_data = utils.open_flex(fcalc_file)
fcalcs_at_en = fcalcs_data["fcalc"]
energies = fcalcs_data["energy"]
return energies, fcalcs_at_en
def __init__(self,
crystal=None,
detector=None,
beam=None,
Ncells_abc=(10, 10, 10),
Gauss=False,
oversample=0,
panel_id=0,
recenter=True,
verbose=10,
profile=None,
device_Id=None,
beamsize_mm=None,
exposure_s=None,
flux=None):
"""
:param crystal: dials crystal model
:param detector: dials detector model
:param beam: dials beam model
"""
self.beam = beam
self.detector = detector
self.panel_id = panel_id
if crystal is None:
crystal = utils.open_flex(cryst_f)
if self.detector is None:
self.detector = utils.open_flex(det_f)
if self.beam is None:
self.beam = utils.open_flex(beam_f)
self.SIM2 = nanoBragg(self.detector,
self.beam,
verbose=verbose,
panel_id=panel_id)
if oversample > 0:
self.SIM2.oversample = oversample
self.SIM2.polarization = 1 # polarization fraction ?
self.SIM2.Ncells_abc = Ncells_abc # important to set this First!
self.SIM2.F000 = 0 # should be number of electrons ?
self.SIM2.default_F = 0
self.SIM2.Amatrix = Amatrix_dials2nanoBragg(
crystal) # sets the unit cell
if Gauss:
self.SIM2.xtal_shape = shapetype.Gauss
else:
self.SIM2.xtal_shape = shapetype.Tophat
if profile is not None: # override above
if profile == "gauss":
self.SIM2.xtal_shape = shapetype.Gauss
elif profile == "tophat":
self.SIM2.xtal_shape = shapetype.Tophat
elif profile == "round":
self.SIM2.xtal_shape = shapetype.Round
elif profile == "square":
self.SIM2.xtal_shape = shapetype.Square
if device_Id is not None:
self.SIM2.device_Id = device_Id
self.SIM2.timelog = False
self.SIM2.progress_meter = False
if flux is not None:
self.SIM2.flux = flux
else:
self.SIM2.flux = 1e14
if beamsize_mm is not None:
self.SIM2.beamsize_mm = beamsize_mm
else:
self.SIM2.beamsize_mm = 0.004
self.SIM2.interpolate = 0
self.SIM2.progress_meter = False
self.SIM2.verbose = verbose
self.SIM2.seed = 9012
self.default_fcalc = None
self.default_interp_en = scattering_factors.interp_energies
self.FULL_ROI = self.SIM2.region_of_interest
if recenter: # FIXME: I am not sure why this seems to be necessary to preserve geom
#a = self.SIM2.beam_center_mm
#print "Beam center was:",a
self.SIM2.beam_center_mm = self.detector[panel_id].get_beam_centre(
self.beam.get_s0())
from dxtbx.model.experiment_list import ExperimentList, Experiment
from dials.array_family import flex
from cxid9114.refine import jitter_refine
from cxid9114.refine import metrics
import scipy.ndimage
import os
# -----------
# Parameters
# -----------
mask_file = "dials_mask_64panels_2.pkl" # mask file for spot detection
img_f = "run62.loc" # dxtbx format file
out_dir = "results.62" # where to dump results
start = int(sys.argv[1])
N = int(sys.argv[2])
DET = utils.open_flex(sys.argv[3])
BEAM = utils.open_flex(sys.argv[4])
tag = sys.argv[5]
# -----------
def load_tracker_f(fname):
data = []
if os.path.exists(fname):
data = np.loadtxt(fname, str)
if data.size and not data.shape:
data = list(set(data[None].astype(int)))
else:
data = list(set(data.astype(int)))
return data
skip_weak = True
dest='noabs',
action='store_true',
help="Plot without taking the absolute value of deviation")
args = parser.parse_args()
cmd = 'find . -name "%s" -type d > _dirs_.txt' % args.iglob
print(cmd)
os.system(cmd)
dirs = np.loadtxt("_dirs_.txt", str)
inits = []
refs = []
for d in dirs:
fnames = glob.glob(os.path.join(d, "*.pkl"))
data = [utils.open_flex(f) for f in fnames]
mxpos = np.argmax([
d['F1'] for d in data
]) # gets the F1 score (agreement between data and simulation)
Cmax = data[mxpos]['crystal']
data_orig = utils.open_flex(fnames[mxpos].split("_refine/")[0] + ".pkl")
Cinit = data_orig["crystalAB"]
img_f = data_orig["img_f"]
loader = dxtbx.load(img_f)
cryst_descr = {
'__id__': 'crystal',
'real_space_a': loader._h5_handle["real_space_a"][()],
'real_space_b': loader._h5_handle["real_space_b"][()],
'real_space_c': loader._h5_handle["real_space_c"][()],
from libtbx.phil import parse
from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color
from cxid9114.refine.test_sim_overlapper import plot_overlap
find_spot_params = find_spots_phil_scope.fetch(source=parse("")).extract()
find_spot_params.spotfinder.threshold.dispersion.global_threshold = 0
find_spot_params.spotfinder.threshold.dispersion.sigma_strong = 0
find_spot_params.spotfinder.filter.min_spot_size = 1
import dxtbx
from dxtbx.datablock import DataBlockFactory
img_f = "/Users/dermen/cxid9114/run62_hits_wtime.h5"
loader = dxtbx.load(img_f)
info_f = utils.open_flex("../index/run62_idx_processed.pkl")
hit_idx = info_f.keys()
idx = hit_idx[0]
iset = loader.get_imageset(img_f)[idx:idx + 1]
dblock = DataBlockFactory.from_imageset(iset)[0]
refls = info_f[idx]['refl']
fracA = info_f[idx]['fracA']
fracB = info_f[idx]['fracB']
cryst_orig = info_f[idx]['crystals'][0]
# load a test crystal
#crystal = utils.open_flex( sim_utils.cryst_f )
# fraction of two color energy
# simulate the pattern
exp_name = sys.argv[1]
data_name = sys.argv[2]
ofile = sys.argv[3]
hkl_tol = .15
run = int(exp_name.split("/")[1].split("run")[1])
shot_idx = int(exp_name.split("_")[1])
ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams
FF = [10000, None] # Setting structure factors takes long time in nanoBragg, so
# unless you want energy-dependent structure factors
# you need only provide one number -or- one structure factor flex miller table
# and the computer will know to preserve that for all beam colors
# load sf for the data, contains wavelength dependence!
FFdat = [utils.open_flex("SA.pkl"), utils.open_flex("SB.pkl")]
FLUX = [1e12, 1e12] # fluxes of the beams
flux_frac = np.random.uniform(.2, .8)
chanA_flux = flux_frac * 1e12
chanB_flux = (1. - flux_frac) * 1e12
FLUXdat = [chanA_flux, chanB_flux]
GAIN = np.random.uniform(0.5, 3)
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
exp_lst = ExperimentListFactory.from_json_file(exp_name) # , check_format=False)
iset = exp_lst.imagesets()[0]
detector = iset.get_detector(0)
data = utils.open_flex(data_name)
import time
import psana
# ------------------
run = int(sys.argv[1])
fix_gain = False
plot = False
odir = "/reg/d/psdm/cxi/cxid9114/res/dermen/minor_fix"
hkl_tol = 0.33
dq_min = 0.005
nom_gain = 28 # nominal photon gain for corrected CSPAD
# ------------------
fnames = glob.glob("results/run%d/*resid.pkl" % run)
mask = [
m.as_numpy_array() for m in utils.open_flex("dials_mask_64panels_2.pkl")
]
spec_df = pandas.read_pickle('ana_result/run%d/run%d_overview_wspec.pdpkl' %
(run, run))
all_spec_hist = []
all_raw_spec = []
loader = dxtbx.load("image_files/_autogen_run%d.loc" % run)
PSANA_ENV = loader.run_mapping[run][2].env() # important to use this env!
# these values in the dataframe represent nominal values, but sometimes
# attenuators were pulled or inserted mid-run, hence
# these are not trustworthy, and we resort to per-event attenuation checks
atten = pandas.read_pickle("atten_cxid9114.pdpkl")
from dxtbx.model.experiment_list import ExperimentList, Experiment
from dials.array_family import flex
from cxid9114.refine import jitter_refine
from cxid9114.refine import metrics
import scipy.ndimage
import os
# -----------
# Parameters
# -----------
mask_file = "dials_mask_64panels_2.pkl" # mask file for spot detection
img_f = sys.argv[1] # dxtbx format image file
out_dir = sys.argv[2] # where to store outpur
start = int(sys.argv[3]) # first shot to process, then proceed
N = int(sys.argv[4]) # number of shots to process
DET = utils.open_flex(sys.argv[5]) # path to pickled detector model
BEAM = utils.open_flex(sys.argv[6]) # '' '' beam model
tag = sys.argv[7] # tag to attache to output files
# -----------
# track shots that indexed, or shots that
# had too few spots to index, so can change parameters and try again
def load_tracker_f(fname):
data = []
if os.path.exists(fname):
data = np.loadtxt(fname, str)
if data.size and not data.shape:
data = list(set(data[None].astype(int)))
else:
data = list(set(data.astype(int)))
return data
from cxid9114 import utils
from cxid9114.sim import sim_utils
from cxid9114.spots import spot_utils
import sys
"""
This script outlines how to compute overlap between strong spots (refls)
and simulated two color images..
"""
# this is a pickle file with at least one crystal and a reflection table in it
data_file = sys.argv[1]
crystal_key = sys.argv[2] # name of crystal in dict
refl_key = sys.argv[3] # name of relfection table in dict
data = utils.open_flex(data_file)
crystal = data[crystal_key]
refls = data[refl_key]
dump = sim_utils.sim_twocolors(crystal,
Ncells_abc=(5, 5, 5),
Gauss=False,
oversample=2)
imgA, imgB = dump['imgA'], dump['imgB']
spotsA = spot_utils.get_spot_data(imgA, thresh=1e-6)
spotsB = spot_utils.get_spot_data(imgB, thresh=1e-6)
spot_utils.plot_overlap(spotsA, spotsB, refls)
" common_mode_algo = pppg\n"
" savgol_polyorder = 3\n"
" mask = d9114_32pan_mask.npy\n"
"}")
with open(img_filename, "w") as oid:
oid.write(img_file)
loader = dxtbx.load(img_filename)
IMGSET = loader.get_imageset(img_filename)
from cxid9114 import mask
mask_file = os.path.dirname(mask.__file__) + "/dials_mask_64panels_2.pkl"
from cxid9114 import geom
geom_folder = os.path.dirname(geom.__file__)
DETECTOR = utils.open_flex(geom_folder + "/ref1_det.pkl")
BEAM = utils.open_flex(geom_folder + "/ref3_beam.pkl")
# --------- spot finding parameter
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from libtbx.phil import parse
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par.spotfinder.threshold.dispersion.global_threshold = 70.
spot_par.spotfinder.threshold.dispersion.gain = 28.
spot_par.spotfinder.threshold.dispersion.kernel_size = [4, 4]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 2.25
spot_par.spotfinder.threshold.dispersion.sigma_background = 6.
spot_par.spotfinder.filter.min_spot_size = 2
spot_par.spotfinder.force_2d = True
spot_par.spotfinder.lookup.mask = mask_file
else:
img_data = np.load(_fpath)["img"]
# get simulation parameters
mos_spread = h5["mos_spread"][()]
Ncells_abc = tuple(h5["Ncells_abc"][()])
mos_doms = h5["mos_doms"][()]
profile = h5["profile"][()]
beamsize = h5["beamsize_mm"][()]
exposure_s = h5["exposure_s"][()]
spectrum = h5["spectrum"][()]
total_flux = np.sum(spectrum)
xtal_size = 0.0005 #h5["xtal_size_mm"][()]
# load reflections
# Note this is used in order to make the strong spot mask to remove spots from background fits
refls_data = utils.open_flex(refl_pkl)
# make a sad spectrum
# FIXME: set an actual miller array where the nonzero elements in GT are
# set to a constant, that way we stay safe from predicting systematic absences
FLUX = [total_flux]
# loading the beam (this might have wrong energy)
BEAM = El.beams()[0]
if args.forcelambda is None:
ave_wave = BEAM.get_wavelength()
else:
ave_wave = args.forcelambda
energies = [parameters.ENERGY_CONV / ave_wave]
DET = El.detectors()[0] # load detector
crystal = El.crystals()[0] # load crystal
from cxid9114 import utils
import glob
from scitbx.matrix import sqr
from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color
fnames = glob.glob("results/good_hit_*_10_crys.pkl")
for fname in fnames:
data = utils.open_flex(fname)
crystal = data['crystal']
xrot = data['optX']
yrot = data['optY']
new_A = xrot * yrot * sqr(crystal.get_U()) * sqr(crystal.get_B())
crystal.set_A(new_A)
orient = indexer_two_color(reflections=refl,
imagesets=[hit_imgset],
params=params)
# h5_name = fname.replace(".pkl", ".h5")
"|F| reference pickle, for computing CCano (only heavy atoms should have anomalous scatering corrections in this reference)"
)
args = parser.parse_args()
from iotbx.reflection_file_reader import any_reflection_file
from cxid9114 import utils
F = any_reflection_file(args.input).as_miller_arrays()[0]
F = F.as_amplitude_array()
utils.save_flex(F, args.output)
print("wrote amplitudes from mtz as a cctbx miller array in file %s " %
args.output)
if args.reference is None:
exit()
# compare with ground truth
ftruth = utils.open_flex(args.reference) # for computing R with ground truth
fobs = F.select(F.resolution_filter_selection(d_max=30, d_min=2.125))
r, c = utils.compute_r_factor(ftruth,
fobs,
verbose=False,
is_flex=True,
sort_flex=True)
ftruth2 = utils.open_flex(args.referenceForCCano) # for comparing to CC ano
r2, c2 = utils.compute_r_factor(ftruth2,
fobs,
verbose=False,
is_flex=True,
sort_flex=True)
print("Rfactor with ground truth: %.2f %%" % (r * 100))
c2 = c + c * j
B2 = sqr((a2, 0, 0, 0, a2, 0, 0, 0, c2)).inverse()
crystal2.set_B(B2)
sim_patt = Patts.make_pattern2(
crystal=crystal2,
flux_per_en=[data['fracA'] * 1e14, data['fracB'] * 1e14],
energies_eV=energy,
fcalcs_at_energies=fcalc,
mosaic_spread=None,
mosaic_domains=None,
ret_sum=True,
Op=None)
sim_sig_mask = sim_patt > 0
overlaps.append(np.sum(sim_sig_mask * spot_mask))
crystals.append(deepcopy(crystal2))
imgs_all.append(sim_patt)
refls_all = [data["refl"]] * len(imgs_all)
utils.images_and_refls_to_simview("cell_refine", imgs_all, refls_all)
return overlaps, crystals
if __name__ == "__main__":
import sys
from cxid9114 import utils
data = utils.open_flex(sys.argv[1])
results, crystals = refine_cell(data)
from IPython import embed
embed()
tag = sys.argv[3]
jitter = int(sys.argv[4])
hkl_tol = .15
ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams
FF = [10000, None] # Setting structure factors takes long time in nanoBragg, so
# unless you want energy-dependent structure factors
# you need only provide one number -or- one structure factor flex miller table
# and the computer will know to preserve that for all beam colors
FLUX = [1e12, 1e12] # fluxes of the beams
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
exp_lst = ExperimentListFactory.from_json_file(exp_name) #, check_format=False)
iset = exp_lst.imagesets()[0]
detector = iset.get_detector(0)
data = utils.open_flex( data_name)
beamA = deepcopy(iset.get_beam())
beamB = deepcopy(iset.get_beam())
beamA.set_wavelength(waveA)
beamB.set_wavelength(waveB)
refls_strong =data["refls_strong"]
crystalAB = data["crystalAB"]
reflsPP = spot_utils.refls_by_panelname(refls_strong)
pids = [i for i in reflsPP if len(reflsPP[i]) > 0] # refine on these panels only
pan_imgs = [iset.get_raw_data(0)[pid].as_numpy_array()
for pid in pids]
#Nper_pid = [ len(reflsPP[i]) for i in pids]
#order_pid = np.argsort( Nper_pid)[::-1]
import sys
try:
tag2 = sys.argv[2]
except IndexError:
tag2 = ""
PIDS = np.arange(32, 40)
spotdata = np.load("crystR.spotdata.pkl.npz")
roi_pp = spotdata['roi_pp'][()]
counts_pp = spotdata["counts_pp"][()]
param_name = sys.argv[1]
params = utils.open_flex(param_name)
tag = os.path.basename(param_name).replace(".pkl", "") + tag2
outdir = tag
if not os.path.exists(outdir):
os.makedirs(outdir)
output_basename = os.path.join(outdir, "simparams_%s" % tag)
h5_fname = output_basename + ".h5py"
h5 = h5py.File(h5_fname, "w")
# load the project beam, crystal-base model, and detector
det = detector = utils.open_flex("xfel_det.pkl")
###########
###########
###########
