def fine_xyrefine(refine_xy_file, n_jobs=6, raw_image=None, use_weights=False): scan_data = utils.open_flex(refine_xy_file) crystal = deepcopy(scan_data['optCrystal']) refl = scan_data['refl'] x = col((1, 0, 0)) y = col((0, 1, 0)) xR = x.axis_and_angle_as_r3_rotation_matrix yR = y.axis_and_angle_as_r3_rotation_matrix fine_degs = np.arange(-0.045, 0.045, 0.005) rotXY_series = [(xR(i, deg=True), yR(j, deg=True)) for i in fine_degs for j in fine_degs] fracA = int(scan_data["fracA"]) fracB = int(scan_data["fracB"]) fcalc_f = scan_data["fcalc_f"] results = xyscan_multi(crystal, fcalc_f, fracA, fracB, refl, rotXY_series, n_jobs=n_jobs, scan_func=xyscan, use_weights=use_weights, raw_image=raw_image) optX_fine, optY_fine = rotXY_series[np.argmax(results)] optA_fine = optX_fine * optY_fine * sqr(crystal.get_U()) * sqr( crystal.get_B()) return results, optX_fine, optY_fine, optA_fine
def from_named_data(pkl_name): from cxid9114 import utils waveA = ENERGY_CONV / 8944. IA = utils.open_flex(pkl_name) # this is refined miller data out = IA.as_mtz_dataset(column_root_label="Iobs", title="B", wavelength=waveA) out.add_miller_array(miller_array=IA.average_bijvoet_mates(), column_root_label="IMEAN") obj = out.mtz_object() obj.write(pkl_name.replace(".pkl", ".mtz"))
def compress_masks(mask_pickles, outname): """ masks compress rather nicely :param mask_pickles: mask pickle files containing dials masks :param outname: hdf5 name """ with h5py.File(outname, "w") as h5: for mask_f in mask_pickles: mask = utils.open_flex(mask_f) # these are tuples in dials land data = [m.as_numpy_array() for m in mask] h5.create_dataset(mask_f, data=data, compression='gzip', compression_opts=9, shuffle=True)
def load_fcalc_for_default_spectrum(): if fcalc_file is None or not os.path.exists(fcalc_file): n_jobs = effective_n_jobs() wavelens_idx = np.array_split(range(len(wavelens_A)), n_jobs) results = Parallel(n_jobs=n_jobs)(delayed(main)(wavelens_A, wavelens_idx[jid], jid) for jid in range(n_jobs)) fcalc_at_wavelen = {} for result_dict in results: for k, v in result_dict.iteritems(): fcalc_at_wavelen.setdefault(k, []).append(v) fcalc_at_wavelen[k] = fcalc_at_wavelen[k][0] else: fcalc_at_wavelen = utils.open_flex(fcalc_file) return fcalc_at_wavelen
def xyscan_multi(crystal, fcalcs_file, fracA, fracB, strong, scan_params, n_jobs, scan_func=xyscan): """ :param crystal_file: :param energies: :param fcalcs_file: :param fracA: :param fracB: :param strong_file: :param rotxy_series: :param n_jobs: :return: """ from joblib import Parallel, delayed nscans = len(scan_params) scan_idx = np.array_split(range(nscans), n_jobs) scan_split = [] for idx in scan_idx: scan_split.append([scan_params[i] for i in idx]) fcalcs_data = utils.open_flex(fcalcs_file) results = Parallel(n_jobs=n_jobs)(delayed(scan_func)\ (crystal=crystal, fcalcs_data=fcalcs_data, fracA=fracA, fracB=fracB, strong=strong,jid=jid, rotxy_series=scan_split[jid]) \ for jid in range(n_jobs)) results = np.concatenate(results) return results
import matplotlib as mpl import time import numpy as np from cxid9114.refine.jitter_refine import JitterFactory from cxid9114.sim import sim_utils import pylab as plt from cxid9114 import parameters from cxid9114 import utils from cxid9114.spots import spot_utils import os import sys from dxtbx.model.detector import Detector from scitbx.array_family import flex # OPEN THE DETECTOR AND THE BEAM! det = detector = utils.open_flex("xfel_det.pkl") beam = utils.open_flex("beam0") #ps2.beam.pkl") #beam = utils.open_flex("ps2.beam.pkl") try: tag = sys.argv[1] outdir = tag if not os.path.exists(outdir): os.makedirs(outdir) except IndexError: tag = "" # DONT YOU EVEN CARE? outdir = "OUTS/" szx = szy = 11 spec_data = np.load("spec_gamma2.npz") ENERGIES = spec_data['en'] - 2000
import cctbx from cctbx import miller from cctbx.crystal import symmetry import os import numpy as np import six import glob from cxid9114 import utils from cctbx.array_family import flex from cctbx import sgtbx from IPython import embed symbol = args.symbol ucell = args.unitcell ftruth = utils.open_flex(args.truthpkl) if six.PY3: asu_map = np.load( os.path.join( args.datdir , "f_asu_map.npy"), allow_pickle=True )[()] else: asu_map = np.load( os.path.join( args.datdir , "f_asu_map.npy") )[()] fcell_pos, asu_indices = zip(*asu_map.items()) flex_indices = flex.miller_index(asu_indices) if args.trial is None: data_files = glob.glob( os.path.join(args.datdir, "_fcell_iter*.npz") ) else: data_files = glob.glob( os.path.join(args.datdir, "_fcell_trial%d_iter*.npz" % args.trial) ) # order files by iteration number
run = int(sys.argv[1]) nom_enA = int(sys.argv[2]) waveA_default = ENERGY_CONV / float(nom_enA) fnames = glob.glob("results/run%d_alpha1alpha2/prepped*dumpsy2" % run) offsets = {} for pid in range(64): offsets[pid] = [] all_f, all_iA, all_iB = [], [], [] waveBs, enBs, del_ens = [], [], [] del_spot_rad, del_spot_abs = [], [] resos = [] for f in fnames: d = utils.open_flex(f) rA = d['residA'] rB = d['residB'] beamA = d['beamA'] beamA.set_wavelength(waveA_default) beamB = d['beamB'] det = d['detector'] refls = d['refls_data'] idxA = rA['indexed'] idxB = rB['indexed'] hkl = rA['hkl'] tree = KDTree(hkl) pairs = tree.query_pairs(1e-3) for i1, i2 in map(list, pairs): if not idxA[i1] and not idxA[i2] and not idxB[i1] and not idxB[i2]: continue
FLUX = [1e12, 1e12] # fluxes of the beams flux_frac = np.random.uniform(.2, .8) chanA_flux = flux_frac * 1e12 chanB_flux = (1. - flux_frac) * 1e12 FLUXdat = [chanA_flux, chanB_flux] GAIN = np.random.uniform(0.5, 3) waveA = parameters.ENERGY_CONV / ENERGIES[0] waveB = parameters.ENERGY_CONV / ENERGIES[1] exp_lst = ExperimentListFactory.from_json_file( exp_name) # , check_format=False) iset = exp_lst.imagesets()[0] detector = iset.get_detector(0) data = utils.open_flex(data_name) beamA = deepcopy(iset.get_beam()) beamB = deepcopy(iset.get_beam()) beamA.set_wavelength(waveA) beamB.set_wavelength(waveB) crystalAB = data["crystalAB"] if has_best: print "Doing the optimized simulation, because you told me to" szx = szy = 5 df = pandas.read_pickle(best_fname) idxmax = df.ave_score.idxmax() best_Amat = tuple(df.Amat.iloc[idxmax]) best_Ncell_abc = tuple(df[["Na", "Nb", "Nc"]].iloc[idxmax].values) best_mos_spread = df["mos_spread"].iloc[idxmax]
degs = np.arange(-0.2, 0.2, 0.025) fine_scan = False use_weights = False # ====================== # END OF PARAMETER ENTRY # ====================== x = col((1., 0., 0.)) y = col((0., 1., 0.)) xRot = x.axis_and_angle_as_r3_rotation_matrix yRot = y.axis_and_angle_as_r3_rotation_matrix rotXY_series = [(xRot(i, deg=True), yRot(j, deg=True)) for i in degs for j in degs] loader = dxtbx.load("/Users/dermen/cxid9114/run62_hits_wtime.h5") data = utils.open_flex(idx_proc_file) hits_w_spec_ana = [ i for i in data if data[i]['spectrum'] is not None and data[i]['can_analyze'] ] if Nprocess > -1: hits_w_spec_ana = hits_w_spec_ana[:Nprocess] for hit_i in hits_w_spec_ana: crystal = data[hit_i]["crystals"][0] refl = data[hit_i]['refl'] fracA = data[hit_i]['fracA'] fracB = data[hit_i]['fracB'] raw_img = loader.get_raw_data(hit_i).as_numpy_array()
#El = ExperimentListFactory.from_json_file(El_json, check_format=False) iset = El.imagesets()[0] fpath = iset.get_path(0) h5 = h5py.File(fpath.replace(".npz", ""), 'r') mos_spread = h5["mos_spread"][()] Ncells_abc = tuple(h5["Ncells_abc"][()]) mos_doms = h5["mos_doms"][()] profile = h5["profile"][()] beamsize = h5["beamsize_mm"][()] exposure_s = h5["exposure_s"][()] spectrum = h5["spectrum"][()] total_flux = np.sum(spectrum) xtal_size = 0.0005 #h5["xtal_size_mm"][()] refls_data = utils.open_flex(refl_pkl) # TODO multi panel # Make a strong spot mask that is used to fit tilting planes FF = [1e3, None] # NOTE: not sure what to do here, we dont know the structure factor FLUX = [total_flux * .5, total_flux*.5] energies = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] if args.sad: FF.pop() FLUX.pop() energies.pop() BEAM = El.beams()[0] DET = El.detectors()[0] crystal = El.crystals()[0] beams = []
from dials.algorithms.indexing.indexer import indexer_base from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from libtbx.phil import parse from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color import dxtbx from dxtbx.datablock import DataBlockFactory from dials.array_family import flex from dxtbx.model import Detector import os fcalc_f = "/Users/dermen/cxid9114_gain/sim/fcalc_slim.pkl" outdir = "ssirp_res_det.refine" if not os.path.exists(outdir): os.makedirs(outdir) BEAM = utils.open_flex(sim_utils.beam_f) sad_wave = parameters.ENERGY_CONV / 8950 BEAM.set_wavelength(sad_wave) MULTI_PANEL = True spot_par = find_spots_phil_scope.fetch(source=parse("")).extract() spot_par_moder = deepcopy(spot_par) spot_par.spotfinder.threshold.dispersion.global_threshold = 70. spot_par.spotfinder.threshold.dispersion.gain = 28. spot_par.spotfinder.threshold.dispersion.kernel_size = [4, 4] spot_par.spotfinder.threshold.dispersion.sigma_strong = 2.25 spot_par.spotfinder.threshold.dispersion.sigma_background = 6. spot_par.spotfinder.filter.min_spot_size = 2 spot_par.spotfinder.force_2d = True spot_par.spotfinder.lookup.mask = "../mask/dials_mask_64panels_2.pkl"
# coding: utf-8 from cxid9114 import utils a=utils.open_flex('SA.pkl') b=utils.open_flex('SB.pkl') a dir(a) a.show_array() b.show_array() ba = b.show_array() ba b.show_array() a.amplitudes() a.amplitudes()[0] dir(a) get_ipython().magic(u'pinfo a.value_at_index') a.value_at_index((1,9,8)) a.indices() a.indices()[0] H = [a.indices()[i] for i in len(a)] H = [a.indices()[i] for i in len(a.indices())] H = [i for i in a.indices()] H a.value_at_index(H[10]) b.value_at_index(H[10]) H[10] a.value_at_index(H[1000]) b.value_at_index(H[1000]) a.resolution_range get_ipython().magic(u'pinfo a.resolution_range') def res_from_idx: def res_from_idx:
def run_sim2smv(Nshot_max, odir, prefix, rank, n_jobs, save_bragg=False, save_smv=True, save_h5 =False, return_pixels=False): from six.moves import range, StringIO from six.moves import cPickle as pickle from cxid9114.sim import sim_utils import os import h5py import math import sys import numpy as np from IPython import embed from cxid9114.bigsim.bigsim_geom import DET,BEAM import scitbx from scitbx.array_family import flex from scitbx.matrix import sqr,col from simtbx.nanoBragg import shapetype from simtbx.nanoBragg import nanoBragg import libtbx.load_env # possibly implicit from libtbx.development.timers import Profiler from cctbx import crystal,crystal_orientation from LS49.sim.step4_pad import microcrystal from cxid9114 import utils from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW from cxid9114.bigsim import sim_spectra odir_j = os.path.join( odir, "job%d" % rank) if not os.path.exists(odir_j): os.makedirs(odir_j) add_noise = False add_background = False overwrite = True #$False sample_thick_mm = 0.005 # 50 micron GDVN nozzle makes a ~5ish micron jet air_thick_mm =0 # mostly vacuum, maybe helium layer of 1 micron flux_ave=2e11 add_spots_algorithm="cuda" big_data = "." # directory location for reference files detpixels_slowfast = (1800,1800) pixsize_mm=0.11 distance_mm = 125 offset_adu=30 mos_spread_deg=0.015 mos_doms=1000 beam_size_mm=0.001 exposure_s=1 use_microcrystal=True #False Ncells_abc=(120,120,120) Deff_A = 2200 length_um = 2.2 timelog = False background = utils.open_flex("background") crystal = microcrystal(Deff_A = Deff_A, length_um = length_um, beam_diameter_um = beam_size_mm*1000, verbose=False) spec_file = h5py.File("simMe_data_run62.h5", "r") spec_data = spec_file["hist_spec"] Umat_data = spec_file["Umats"] en_chans = spec_file["energy_bins"][()] ilow = np.abs(en_chans - ENERGY_LOW).argmin() ihigh = np.abs(en_chans - ENERGY_HIGH).argmin() wave_chans = ENERGY_CONV/en_chans sfall_main = sim_spectra.load_spectra("test_sfall.h5") Nshot = spec_data.shape[0] idx_range = np.array_split(np.arange(Nshot), n_jobs) Nshot_per_job = len(idx_range[rank]) if Nshot_max > 0 : Nshot_per_job = min( Nshot_max, Nshot_per_job) print ("Job %d: Simulating %d shots" % (rank, Nshot_per_job)) istart = idx_range[rank][0] istop = istart + Nshot_per_job smi_stride = 10 for idx in range( istart, istop): print ("<><><><><><><><><><><><><><>") print ("Job %d; Image %d (%d - %d)" % (rank, idx+1, istart, istop)) print ("<><><><><><><><><><><><><><>") smv_fileout = os.path.join( odir_j, prefix % idx + ".img") h5_fileout = smv_fileout + ".h5" if os.path.exists(smv_fileout) and not overwrite and save_smv: print("Shot %s exists: moving on" % smv_fileout) continue if os.path.exists(h5_fileout) and not overwrite and save_h5: print("Shot %s exists: moving on" % h5_fileout) continue if (rank==0 and idx % smi_stride==0): print("GPU status") os.system("nvidia-smi") print("\n\n") print("CPU memory usage") mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'""" os.system(mem_usg) spec = spec_data[2] rotation = sqr(Umat_data[2]) wavelength_A = np.mean(wave_chans) spectra = iter([(wave_chans, spec, wavelength_A)]) direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength assert wavelength_A > 0 assert (len(wavlen)==len(flux)==len(sfall_main)) N = crystal.number_of_cells(sfall_main[0].unit_cell()) if use_mcrocrystal: Ncells_abc = (N,N,N) flux *= 0 flux[ilow] = 1e12 flux[ihigh]=1e12 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mos_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(mos_doms) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) Amatrix_rot = (rotation * sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose() #SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast, # pixel_size_mm=pixsize_mm,Ncells_abc=Ncells_abc, # wavelength_A=wavelength_A,verbose=verbose) SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose) SIM.adc_offset_adu = offset_adu # Do not offset by 40 SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = mos_doms SIM.distance_mm=distance_mm SIM.set_mosaic_blocks(UMAT_nm) SIM.seed = 1 SIM.oversample=1 SIM.wavelength_A = wavelength_A SIM.polarization=1 SIM.default_F=0 SIM.Fhkl=sfall_main[0].amplitudes() SIM.progress_meter=False SIM.flux=flux_ave SIM.exposure_s=exposure_s SIM.beamsize_mm=beam_size_mm SIM.Ncells_abc=Ncells_abc SIM.xtal_shape=shapetype.Gauss idxpath = "try3_idx2/job0/dump_0_data.pkl" Amat = sim_utils.Amatrix_dials2nanoBragg(utils.open_flex(idxpath)["crystalAB"]) #SIM.Umatrix = rotation.elems #SIM.unit_cell_Adeg = sfall_main[0].unit_cell() #SIM2 = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose) #SIM2.adc_offset_adu = offset_adu # Do not offset by 40 #SIM2.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev #SIM2.mosaic_domains = mos_doms #SIM2.distance_mm=distance_mm #SIM2.set_mosaic_blocks(UMAT_nm) #SIM2.seed = 1 #SIM2.oversample=1 #SIM2.wavelength_A = wavelength_A #SIM2.polarization=1 #SIM2.default_F=0 #SIM2.Fhkl=sfall_main[0].amplitudes() #SIM2.progress_meter=False #SIM2.flux=flux_ave #SIM2.exposure_s=exposure_s #SIM2.beamsize_mm=beam_size_mm #SIM2.Ncells_abc=Ncells_abc #SIM2.xtal_shape=shapetype.Gauss #SIM2.Umatrix = rotation.elems #SIM2.unit_cell_Adeg = sfall_main[0].unit_cell() #SIM.Amatrix_RUB = Amatrix_rot #SIM.Amatrix = Amatrix_rot.inverse() raw_pixel_sum = flex.double(len(SIM.raw_pixels)) Nflux = len(flux) print("Beginning the loop") for x in range(Nflux): if x % 10==0: print("+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" % (x+1, Nflux) , end="\r") if flux[x] ==0: continue #SIM.Amatrix = Amatrix_rot.inverse() print (SIM.Ncells_abc) SIM.wavelength_A=wavlen[x] SIM.flux=flux[x] SIM.Fhkl=sfall_main[x].amplitudes() SIM.Amatrix = Amat#Amatrix_rot.inverse() #sim_utils.compare_sims(SIM, SIM2) #SIM.Ncells_abc=Ncells_abc #SIM.adc_offset_adu = offset_adu #SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev #SIM.mosaic_domains = mos_doms # #SIM.distance_mm=distance_mm #SIM.set_mosaic_blocks(UMAT_nm) #SIM.seed = 1 #SIM.polarization=1 #SIM.default_F=0 #SIM.xtal_shape=shapetype.Gauss #SIM.progress_meter=False #SIM.exposure_s = exposure_s #SIM.beamsize_mm=beam_size_mm SIM.timelog=timelog SIM.device_Id=rank SIM.raw_pixels *= 0 # just in case! SIM.add_nanoBragg_spots_cuda() if use_microcrystal: raw_pixel_sum += SIM.raw_pixels * crystal.domains_per_crystal print() SIM.raw_pixels = raw_pixel_sum if add_background: SIM.raw_pixels = SIM.raw_pixels + background SIM.detector_psf_kernel_radius_pixels=5; #SIM.detector_psf_fwhm_mm=0.08; #SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian) SIM.detector_psf_type=shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm=0 SIM.quantum_gain = 28. #SIM.apply_psf() if add_noise: SIM.add_noise() #converts phtons to ADU. extra = "PREFIX=%s;\nRANK=%d;\n"%(prefix,rank) out = SIM.raw_pixels.as_numpy_array() if save_smv: SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True) elif save_h5: f = h5py.File(h5_fileout, "w") f.create_dataset("bigsim_d9114", data=SIM.raw_pixels.as_numpy_array().astype(np.uint16).reshape(detpixels_slowfast), compression="lzf") f.close() if npout is not None: np.save(npout, SIM.raw_pixels.as_numpy_array()) SIM.free_all()
def load_fcalc_file(fcalc_file): fcalcs_data = utils.open_flex(fcalc_file) fcalcs_at_en = fcalcs_data["fcalc"] energies = fcalcs_data["energy"] return energies, fcalcs_at_en
def __init__(self, crystal=None, detector=None, beam=None, Ncells_abc=(10, 10, 10), Gauss=False, oversample=0, panel_id=0, recenter=True, verbose=10, profile=None, device_Id=None, beamsize_mm=None, exposure_s=None, flux=None): """ :param crystal: dials crystal model :param detector: dials detector model :param beam: dials beam model """ self.beam = beam self.detector = detector self.panel_id = panel_id if crystal is None: crystal = utils.open_flex(cryst_f) if self.detector is None: self.detector = utils.open_flex(det_f) if self.beam is None: self.beam = utils.open_flex(beam_f) self.SIM2 = nanoBragg(self.detector, self.beam, verbose=verbose, panel_id=panel_id) if oversample > 0: self.SIM2.oversample = oversample self.SIM2.polarization = 1 # polarization fraction ? self.SIM2.Ncells_abc = Ncells_abc # important to set this First! self.SIM2.F000 = 0 # should be number of electrons ? self.SIM2.default_F = 0 self.SIM2.Amatrix = Amatrix_dials2nanoBragg( crystal) # sets the unit cell if Gauss: self.SIM2.xtal_shape = shapetype.Gauss else: self.SIM2.xtal_shape = shapetype.Tophat if profile is not None: # override above if profile == "gauss": self.SIM2.xtal_shape = shapetype.Gauss elif profile == "tophat": self.SIM2.xtal_shape = shapetype.Tophat elif profile == "round": self.SIM2.xtal_shape = shapetype.Round elif profile == "square": self.SIM2.xtal_shape = shapetype.Square if device_Id is not None: self.SIM2.device_Id = device_Id self.SIM2.timelog = False self.SIM2.progress_meter = False if flux is not None: self.SIM2.flux = flux else: self.SIM2.flux = 1e14 if beamsize_mm is not None: self.SIM2.beamsize_mm = beamsize_mm else: self.SIM2.beamsize_mm = 0.004 self.SIM2.interpolate = 0 self.SIM2.progress_meter = False self.SIM2.verbose = verbose self.SIM2.seed = 9012 self.default_fcalc = None self.default_interp_en = scattering_factors.interp_energies self.FULL_ROI = self.SIM2.region_of_interest if recenter: # FIXME: I am not sure why this seems to be necessary to preserve geom #a = self.SIM2.beam_center_mm #print "Beam center was:",a self.SIM2.beam_center_mm = self.detector[panel_id].get_beam_centre( self.beam.get_s0())
from dxtbx.model.experiment_list import ExperimentList, Experiment from dials.array_family import flex from cxid9114.refine import jitter_refine from cxid9114.refine import metrics import scipy.ndimage import os # ----------- # Parameters # ----------- mask_file = "dials_mask_64panels_2.pkl" # mask file for spot detection img_f = "run62.loc" # dxtbx format file out_dir = "results.62" # where to dump results start = int(sys.argv[1]) N = int(sys.argv[2]) DET = utils.open_flex(sys.argv[3]) BEAM = utils.open_flex(sys.argv[4]) tag = sys.argv[5] # ----------- def load_tracker_f(fname): data = [] if os.path.exists(fname): data = np.loadtxt(fname, str) if data.size and not data.shape: data = list(set(data[None].astype(int))) else: data = list(set(data.astype(int))) return data skip_weak = True
dest='noabs', action='store_true', help="Plot without taking the absolute value of deviation") args = parser.parse_args() cmd = 'find . -name "%s" -type d > _dirs_.txt' % args.iglob print(cmd) os.system(cmd) dirs = np.loadtxt("_dirs_.txt", str) inits = [] refs = [] for d in dirs: fnames = glob.glob(os.path.join(d, "*.pkl")) data = [utils.open_flex(f) for f in fnames] mxpos = np.argmax([ d['F1'] for d in data ]) # gets the F1 score (agreement between data and simulation) Cmax = data[mxpos]['crystal'] data_orig = utils.open_flex(fnames[mxpos].split("_refine/")[0] + ".pkl") Cinit = data_orig["crystalAB"] img_f = data_orig["img_f"] loader = dxtbx.load(img_f) cryst_descr = { '__id__': 'crystal', 'real_space_a': loader._h5_handle["real_space_a"][()], 'real_space_b': loader._h5_handle["real_space_b"][()], 'real_space_c': loader._h5_handle["real_space_c"][()],
from libtbx.phil import parse from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color from cxid9114.refine.test_sim_overlapper import plot_overlap find_spot_params = find_spots_phil_scope.fetch(source=parse("")).extract() find_spot_params.spotfinder.threshold.dispersion.global_threshold = 0 find_spot_params.spotfinder.threshold.dispersion.sigma_strong = 0 find_spot_params.spotfinder.filter.min_spot_size = 1 import dxtbx from dxtbx.datablock import DataBlockFactory img_f = "/Users/dermen/cxid9114/run62_hits_wtime.h5" loader = dxtbx.load(img_f) info_f = utils.open_flex("../index/run62_idx_processed.pkl") hit_idx = info_f.keys() idx = hit_idx[0] iset = loader.get_imageset(img_f)[idx:idx + 1] dblock = DataBlockFactory.from_imageset(iset)[0] refls = info_f[idx]['refl'] fracA = info_f[idx]['fracA'] fracB = info_f[idx]['fracB'] cryst_orig = info_f[idx]['crystals'][0] # load a test crystal #crystal = utils.open_flex( sim_utils.cryst_f ) # fraction of two color energy # simulate the pattern
exp_name = sys.argv[1] data_name = sys.argv[2] ofile = sys.argv[3] hkl_tol = .15 run = int(exp_name.split("/")[1].split("run")[1]) shot_idx = int(exp_name.split("_")[1]) ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams FF = [10000, None] # Setting structure factors takes long time in nanoBragg, so # unless you want energy-dependent structure factors # you need only provide one number -or- one structure factor flex miller table # and the computer will know to preserve that for all beam colors # load sf for the data, contains wavelength dependence! FFdat = [utils.open_flex("SA.pkl"), utils.open_flex("SB.pkl")] FLUX = [1e12, 1e12] # fluxes of the beams flux_frac = np.random.uniform(.2, .8) chanA_flux = flux_frac * 1e12 chanB_flux = (1. - flux_frac) * 1e12 FLUXdat = [chanA_flux, chanB_flux] GAIN = np.random.uniform(0.5, 3) waveA = parameters.ENERGY_CONV / ENERGIES[0] waveB = parameters.ENERGY_CONV / ENERGIES[1] exp_lst = ExperimentListFactory.from_json_file(exp_name) # , check_format=False) iset = exp_lst.imagesets()[0] detector = iset.get_detector(0) data = utils.open_flex(data_name)
import time import psana # ------------------ run = int(sys.argv[1]) fix_gain = False plot = False odir = "/reg/d/psdm/cxi/cxid9114/res/dermen/minor_fix" hkl_tol = 0.33 dq_min = 0.005 nom_gain = 28 # nominal photon gain for corrected CSPAD # ------------------ fnames = glob.glob("results/run%d/*resid.pkl" % run) mask = [ m.as_numpy_array() for m in utils.open_flex("dials_mask_64panels_2.pkl") ] spec_df = pandas.read_pickle('ana_result/run%d/run%d_overview_wspec.pdpkl' % (run, run)) all_spec_hist = [] all_raw_spec = [] loader = dxtbx.load("image_files/_autogen_run%d.loc" % run) PSANA_ENV = loader.run_mapping[run][2].env() # important to use this env! # these values in the dataframe represent nominal values, but sometimes # attenuators were pulled or inserted mid-run, hence # these are not trustworthy, and we resort to per-event attenuation checks atten = pandas.read_pickle("atten_cxid9114.pdpkl")
from dxtbx.model.experiment_list import ExperimentList, Experiment from dials.array_family import flex from cxid9114.refine import jitter_refine from cxid9114.refine import metrics import scipy.ndimage import os # ----------- # Parameters # ----------- mask_file = "dials_mask_64panels_2.pkl" # mask file for spot detection img_f = sys.argv[1] # dxtbx format image file out_dir = sys.argv[2] # where to store outpur start = int(sys.argv[3]) # first shot to process, then proceed N = int(sys.argv[4]) # number of shots to process DET = utils.open_flex(sys.argv[5]) # path to pickled detector model BEAM = utils.open_flex(sys.argv[6]) # '' '' beam model tag = sys.argv[7] # tag to attache to output files # ----------- # track shots that indexed, or shots that # had too few spots to index, so can change parameters and try again def load_tracker_f(fname): data = [] if os.path.exists(fname): data = np.loadtxt(fname, str) if data.size and not data.shape: data = list(set(data[None].astype(int))) else: data = list(set(data.astype(int))) return data
from cxid9114 import utils from cxid9114.sim import sim_utils from cxid9114.spots import spot_utils import sys """ This script outlines how to compute overlap between strong spots (refls) and simulated two color images.. """ # this is a pickle file with at least one crystal and a reflection table in it data_file = sys.argv[1] crystal_key = sys.argv[2] # name of crystal in dict refl_key = sys.argv[3] # name of relfection table in dict data = utils.open_flex(data_file) crystal = data[crystal_key] refls = data[refl_key] dump = sim_utils.sim_twocolors(crystal, Ncells_abc=(5, 5, 5), Gauss=False, oversample=2) imgA, imgB = dump['imgA'], dump['imgB'] spotsA = spot_utils.get_spot_data(imgA, thresh=1e-6) spotsB = spot_utils.get_spot_data(imgB, thresh=1e-6) spot_utils.plot_overlap(spotsA, spotsB, refls)
" common_mode_algo = pppg\n" " savgol_polyorder = 3\n" " mask = d9114_32pan_mask.npy\n" "}") with open(img_filename, "w") as oid: oid.write(img_file) loader = dxtbx.load(img_filename) IMGSET = loader.get_imageset(img_filename) from cxid9114 import mask mask_file = os.path.dirname(mask.__file__) + "/dials_mask_64panels_2.pkl" from cxid9114 import geom geom_folder = os.path.dirname(geom.__file__) DETECTOR = utils.open_flex(geom_folder + "/ref1_det.pkl") BEAM = utils.open_flex(geom_folder + "/ref3_beam.pkl") # --------- spot finding parameter from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from libtbx.phil import parse spot_par = find_spots_phil_scope.fetch(source=parse("")).extract() spot_par.spotfinder.threshold.dispersion.global_threshold = 70. spot_par.spotfinder.threshold.dispersion.gain = 28. spot_par.spotfinder.threshold.dispersion.kernel_size = [4, 4] spot_par.spotfinder.threshold.dispersion.sigma_strong = 2.25 spot_par.spotfinder.threshold.dispersion.sigma_background = 6. spot_par.spotfinder.filter.min_spot_size = 2 spot_par.spotfinder.force_2d = True spot_par.spotfinder.lookup.mask = mask_file
else: img_data = np.load(_fpath)["img"] # get simulation parameters mos_spread = h5["mos_spread"][()] Ncells_abc = tuple(h5["Ncells_abc"][()]) mos_doms = h5["mos_doms"][()] profile = h5["profile"][()] beamsize = h5["beamsize_mm"][()] exposure_s = h5["exposure_s"][()] spectrum = h5["spectrum"][()] total_flux = np.sum(spectrum) xtal_size = 0.0005 #h5["xtal_size_mm"][()] # load reflections # Note this is used in order to make the strong spot mask to remove spots from background fits refls_data = utils.open_flex(refl_pkl) # make a sad spectrum # FIXME: set an actual miller array where the nonzero elements in GT are # set to a constant, that way we stay safe from predicting systematic absences FLUX = [total_flux] # loading the beam (this might have wrong energy) BEAM = El.beams()[0] if args.forcelambda is None: ave_wave = BEAM.get_wavelength() else: ave_wave = args.forcelambda energies = [parameters.ENERGY_CONV / ave_wave] DET = El.detectors()[0] # load detector crystal = El.crystals()[0] # load crystal
from cxid9114 import utils import glob from scitbx.matrix import sqr from cxi_xdr_xes.two_color.two_color_indexer import indexer_two_color fnames = glob.glob("results/good_hit_*_10_crys.pkl") for fname in fnames: data = utils.open_flex(fname) crystal = data['crystal'] xrot = data['optX'] yrot = data['optY'] new_A = xrot * yrot * sqr(crystal.get_U()) * sqr(crystal.get_B()) crystal.set_A(new_A) orient = indexer_two_color(reflections=refl, imagesets=[hit_imgset], params=params) # h5_name = fname.replace(".pkl", ".h5")
"|F| reference pickle, for computing CCano (only heavy atoms should have anomalous scatering corrections in this reference)" ) args = parser.parse_args() from iotbx.reflection_file_reader import any_reflection_file from cxid9114 import utils F = any_reflection_file(args.input).as_miller_arrays()[0] F = F.as_amplitude_array() utils.save_flex(F, args.output) print("wrote amplitudes from mtz as a cctbx miller array in file %s " % args.output) if args.reference is None: exit() # compare with ground truth ftruth = utils.open_flex(args.reference) # for computing R with ground truth fobs = F.select(F.resolution_filter_selection(d_max=30, d_min=2.125)) r, c = utils.compute_r_factor(ftruth, fobs, verbose=False, is_flex=True, sort_flex=True) ftruth2 = utils.open_flex(args.referenceForCCano) # for comparing to CC ano r2, c2 = utils.compute_r_factor(ftruth2, fobs, verbose=False, is_flex=True, sort_flex=True) print("Rfactor with ground truth: %.2f %%" % (r * 100))
c2 = c + c * j B2 = sqr((a2, 0, 0, 0, a2, 0, 0, 0, c2)).inverse() crystal2.set_B(B2) sim_patt = Patts.make_pattern2( crystal=crystal2, flux_per_en=[data['fracA'] * 1e14, data['fracB'] * 1e14], energies_eV=energy, fcalcs_at_energies=fcalc, mosaic_spread=None, mosaic_domains=None, ret_sum=True, Op=None) sim_sig_mask = sim_patt > 0 overlaps.append(np.sum(sim_sig_mask * spot_mask)) crystals.append(deepcopy(crystal2)) imgs_all.append(sim_patt) refls_all = [data["refl"]] * len(imgs_all) utils.images_and_refls_to_simview("cell_refine", imgs_all, refls_all) return overlaps, crystals if __name__ == "__main__": import sys from cxid9114 import utils data = utils.open_flex(sys.argv[1]) results, crystals = refine_cell(data) from IPython import embed embed()
tag = sys.argv[3] jitter = int(sys.argv[4]) hkl_tol = .15 ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams FF = [10000, None] # Setting structure factors takes long time in nanoBragg, so # unless you want energy-dependent structure factors # you need only provide one number -or- one structure factor flex miller table # and the computer will know to preserve that for all beam colors FLUX = [1e12, 1e12] # fluxes of the beams waveA = parameters.ENERGY_CONV / ENERGIES[0] waveB = parameters.ENERGY_CONV / ENERGIES[1] exp_lst = ExperimentListFactory.from_json_file(exp_name) #, check_format=False) iset = exp_lst.imagesets()[0] detector = iset.get_detector(0) data = utils.open_flex( data_name) beamA = deepcopy(iset.get_beam()) beamB = deepcopy(iset.get_beam()) beamA.set_wavelength(waveA) beamB.set_wavelength(waveB) refls_strong =data["refls_strong"] crystalAB = data["crystalAB"] reflsPP = spot_utils.refls_by_panelname(refls_strong) pids = [i for i in reflsPP if len(reflsPP[i]) > 0] # refine on these panels only pan_imgs = [iset.get_raw_data(0)[pid].as_numpy_array() for pid in pids] #Nper_pid = [ len(reflsPP[i]) for i in pids] #order_pid = np.argsort( Nper_pid)[::-1]
import sys try: tag2 = sys.argv[2] except IndexError: tag2 = "" PIDS = np.arange(32, 40) spotdata = np.load("crystR.spotdata.pkl.npz") roi_pp = spotdata['roi_pp'][()] counts_pp = spotdata["counts_pp"][()] param_name = sys.argv[1] params = utils.open_flex(param_name) tag = os.path.basename(param_name).replace(".pkl", "") + tag2 outdir = tag if not os.path.exists(outdir): os.makedirs(outdir) output_basename = os.path.join(outdir, "simparams_%s" % tag) h5_fname = output_basename + ".h5py" h5 = h5py.File(h5_fname, "w") # load the project beam, crystal-base model, and detector det = detector = utils.open_flex("xfel_det.pkl") ########### ########### ###########