def unload_compress_masks(fname): """ unpacks the fname and stores the masks in dials format :param fname: a compress_mask files created using compress_masks """ with h5py.File(fname, "r") as h5: for name in h5.keys(): if os.path.exists(name): print("Path %s exists!, skipping" % name) continue data = h5[name][()] if data.dtype != bool: continue data = tuple([flex.bool(m) for m in data]) utils.save_flex(data, name)
print 'adding in anom scattering for yb' for i, s in enumerate(sc): if i in yb_pos: fp, fdp = scattering_factors.elem_fp_fdp_at_eV(s.element_symbol(), args.e, how='henke') s.fp = fp s.fdp = fdp #fp, fdp = scattering_factors.Yb_fp_fdp_at_eV(energy_eV, 'henke') #sc[yb_pos[0]].fp = fp #sc[yb_pos[0]].fdp = fdp #sc[yb_pos[1]].fp = fp #sc[yb_pos[1]].fdp = fdp #xr2_sel = flex.bool(len(sc), True) #for pos in yb_pos: # xr2_sel[pos] = False #xr2 = xr.select(xr2_sel) Ftot = xr.structure_factors(d_min=d_min, algorithm='direct', anomalous_flag=True).f_calc() Atot = Ftot.data().as_numpy_array() #print "Ftotal and Fprot + Fheavy should be equalish!" #assert (np.allclose(Atot, Aa+Ap)) out = {"Aprotein": Fp, "Aheavy": Fa, "Atotal": Ftot} utils.save_flex(out, args.o)
def msi(jid): """ msi stands for mad --> spot --> index Here we load a data image that has crystal diffraction then we spot peaks and index the crystal image """ #time.sleep(jid) import sys import numpy as np import glob import os from copy import deepcopy from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dials.command_line.stills_process import phil_scope\ as indexer_phil_scope from dials.algorithms.indexing.lattice_search import basis_vector_search_phil_scope from dials.array_family import flex import dxtbx from dxtbx.datablock import DataBlockFactory from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList, Experiment from libtbx.utils import Sorry as Sorry from libtbx.phil import parse from cxi_xdr_xes.command_line.two_color_process import two_color_phil_scope from cxi_xdr_xes.two_color import two_color_indexer indexer_two_color = two_color_indexer.indexer_two_color from cctbx import crystal from cxid9114 import utils from cxid9114 import parameters indexer_phil_scope.adopt_scope(two_color_phil_scope) indexer_phil_scope.adopt_scope(basis_vector_search_phil_scope) mad_index_params = indexer_phil_scope.extract() fnames = glob.glob(glob_str) out_dir = os.path.join( args.out_dir , "job%d" % jid) if not os.path.exists(out_dir): os.makedirs(out_dir) refls_dir = "refls_%s" % tag Els_dir = "Els_%s" % tag dumps_dir = "dumps_%s" % tag for o in [refls_dir, Els_dir, dumps_dir]: sub_o = os.path.join(out_dir, o) if not os.path.exists(sub_o): os.makedirs(sub_o) # track shots that indexed, or shots that # had too few spots to index, so can change parameters and try again def load_tracker_f(fname): data = [] if os.path.exists(fname): data = np.loadtxt(fname, str) if data.size and not data.shape: data = list(set(data[None].astype(int))) else: data = list(set(data.astype(int))) return data skip_weak = False skip_failed = True #False skip_indexed = False weak_shots_f = os.path.join(out_dir, "weak_shots.txt") failed_idx_f = os.path.join(out_dir, "failed_shots.txt") indexed_f = os.path.join(out_dir, "indexed_shots.txt") weak_shots = load_tracker_f(weak_shots_f) failed_shots = load_tracker_f(failed_idx_f) indexed_shots = load_tracker_f(indexed_f) if not os.path.exists(out_dir): os.makedirs(out_dir) # ------ spotting parameters spot_par = find_spots_phil_scope.fetch(source=parse("")).extract() spot_par.spotfinder.threshold.dispersion.global_threshold = 40 spot_par.spotfinder.threshold.dispersion.gain = 28 spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2] spot_par.spotfinder.threshold.dispersion.sigma_strong = 1 spot_par.spotfinder.threshold.dispersion.sigma_background = 6 spot_par.spotfinder.filter.min_spot_size = 3 spot_par.spotfinder.force_2d = True # ------ indexing parameters KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P1") #KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P43212") mad_index_params.refinement.parameterisation.beam.fix = "all" mad_index_params.refinement.parameterisation.detector.fix = "all" #mad_index_params.refinement.verbosity = 3 #mad_index_params.refinement.reflections.outlier.algorithm = "null" mad_index_params.indexing.stills.refine_all_candidates = do_stills_refine mad_index_params.indexing.optimise_initial_basis_vectors = do_basis_refine #mad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False mad_index_params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info() mad_index_params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell() mad_index_params.indexing.refinement_protocol.d_min_start = None mad_index_params.indexing.debug = True mad_index_params.indexing.real_space_grid_search.characteristic_grid = 0.02 mad_index_params.indexing.known_symmetry.absolute_angle_tolerance = 5.0 mad_index_params.indexing.known_symmetry.relative_length_tolerance = 0.3 mad_index_params.indexing.stills.rmsd_min_px = 20000 mad_index_params.indexing.refinement_protocol.n_macro_cycles = 1 mad_index_params.indexing.multiple_lattice_search.max_lattices = 1 mad_index_params.indexing.basis_vector_combinations.max_refine = 10000000000 #mad_index_params.indexing.basis_vector_combinations.max_combinations = 150 #mad_index_params.indexing.stills.candidate_outlier_rejection = False mad_index_params.indexing.refinement_protocol.mode = None #$"repredict_only" mad_index_params.indexing.two_color.high_energy = parameters.ENERGY_HIGH mad_index_params.indexing.two_color.low_energy = parameters.ENERGY_LOW mad_index_params.indexing.two_color.avg_energy = parameters.ENERGY_LOW * .5 + parameters.ENERGY_HIGH * .5 #mad_index_params.indexing.two_color.spiral_method = (1., 100000) # 1000000) mad_index_params.indexing.two_color.n_unique_v = 22 #mad_index_params.indexing.two_color.block_size = 25 #mad_index_params.indexing.two_color.filter_by_mag = (10,3) # ------ N = len(fnames) idx_split = np.array_split(np.arange(N), n_jobs) n_idx = 0 # number indexed for idx in idx_split[jid]: if args.Nmax is not None: if idx == args.Nmax: sys.exit() img_f = fnames[idx] loader = dxtbx.load(img_f) iset = loader.get_imageset(filenames=[loader.get_image_file()] ) DET = loader.get_detector() BEAM = loader.get_beam() El = ExperimentListFactory.from_imageset_and_crystal(iset, crystal=None) E = El[0] from cxid9114.geom.multi_panel import CSPAD_refined, CSPAD El[0].detector = CSPAD_refined if idx in weak_shots and skip_weak: print("Skipping weak shots %d" % idx) continue if idx in failed_shots and skip_failed: print("Skipping failed idx shots %d" % idx) continue if idx in indexed_shots and skip_indexed: print("Skipping already idx shots %d" % idx) continue refls_strong = flex.reflection_table.from_observations(El, spot_par) if len(refls_strong) < min_spot_per_pattern: print("Not enough spots shot %d, continuing!" % idx) weak_shots.append(idx) try: np.savetxt(weak_shots_f, weak_shots, fmt="%d") except: pass continue # ================================== # 2 color indexer of 2 color pattern # ================================== try: orientAB = indexer_two_color( reflections=refls_strong, experiments=El, params=mad_index_params) orientAB.index() except (Sorry, RuntimeError, AssertionError) as error: print("####\nIndexingFailed T_T \n####") print (error) failed_shots.append(idx) try: np.savetxt(failed_idx_f, failed_shots, fmt="%d") except: pass continue print("####\n *_* IndexingWORKED %d *_* \n####" % idx) n_idx += 1 indexed_shots.append(idx) try: np.savetxt(indexed_f, indexed_shots, fmt="%d") except: pass crystalAB = orientAB.refined_experiments.crystals()[0] refl_pkl = os.path.join(out_dir,refls_dir, "refl_%d_%s.pkl" % (idx, tag)) utils.save_flex(refls_strong, refl_pkl) El[0].crystal = crystalAB El_json = os.path.join(out_dir, Els_dir, "El_%d_%s.json" % (idx, tag)) El.as_json(filename=El_json)
if args.lastonly: indices = [indices[-1]] sgi = sgtbx.space_group_info(symbol) sym = symmetry(unit_cell=ucell, space_group_info=sgi) mset = miller.set(sym , flex_indices, anomalous_flag=True) for i_iter in indices: if six.PY3: fcell_data = flex.double( np.load(data_files[i_iter], allow_pickle=True)['fvals'] ) else: fcell_data = flex.double( np.load(data_files[i_iter])['fvals'] ) fobs = miller.array( mset, data=fcell_data).set_observation_type_xray_amplitude() if args.savelast is not None and i_iter==indices[-1]: utils.save_flex(fobs, args.savelast) fobs_sel = fobs.select(fobs.resolution_filter_selection(d_max=30, d_min=args.dmin)) if args.binning: r,c = utils.compute_r_factor_binned( ftruth, fobs_sel, verbose=True, d_max=30, d_min=args.dmin, n_bin=10) r,c = utils.compute_r_factor( ftruth, fobs_sel, verbose=False, is_flex=True, sort_flex=True) print ("Iter %d: Overall: Rtruth=%.4f, CCdeltaAnom=%.4f"% (i_iter, r,c)) else: r,c = utils.compute_r_factor( ftruth, fobs_sel, verbose=False, is_flex=True, sort_flex=True) print ("iter %d: Rtruth=%.4f, CCdeltaAnom=%.4f"% (i_iter, r,c)) print (fobs_sel.completeness()) if args.mtzoutput is not None: fobs.as_mtz_dataset("Fobs").mtz_object().write(args.mtzoutput)
data2 = { "dQ_AnotB": all_qAnotB, "dQ_BnotA": all_qBnotA, "dQ_AandB": all_qAandB, "dQ_AorB": all_qAorB, "dij_AnotB": all_dAnotB, "dij_BnotA": all_dBnotA, "dij_AandB": all_dAandB, "dij_AorB": all_dAorB, "res_AnotB": all_resAnotB, "res_BnotA": all_resBnotA, "res_AandB": all_resAandB, "res_AorB": all_resAorB, } utils.save_flex(data2, odir + "/" + "run%d_details.pkl" % run) dist_out3 = np.array(dist_out3) cols = [ "numclose", "rmsd_v1", "fname", "run_num", "shot_idx", "frac_indexed", "Nref", "Nindexed", "AnotB", "BnotA", "AandB", "AorB" ] dtypes = [ np.int32, np.float32, str, np.int32, np.int32, np.float32, np.int32, np.int32, np.int32, np.int32, np.int32, np.int32 ] df = pandas.DataFrame(dist_out3[:, [0] + range(2, 13)], columns=cols) for i, col in enumerate(cols): df[col] = df[col].astype(dtypes[i]) df.to_pickle(odir + "/" + "run%d_overview.pdpkl" % run)
default=None, help="|F| reference pickle") parser.add_argument( "--referenceForCCano", type=str, default=None, help= "|F| reference pickle, for computing CCano (only heavy atoms should have anomalous scatering corrections in this reference)" ) args = parser.parse_args() from iotbx.reflection_file_reader import any_reflection_file from cxid9114 import utils F = any_reflection_file(args.input).as_miller_arrays()[0] F = F.as_amplitude_array() utils.save_flex(F, args.output) print("wrote amplitudes from mtz as a cctbx miller array in file %s " % args.output) if args.reference is None: exit() # compare with ground truth ftruth = utils.open_flex(args.reference) # for computing R with ground truth fobs = F.select(F.resolution_filter_selection(d_max=30, d_min=2.125)) r, c = utils.compute_r_factor(ftruth, fobs, verbose=False, is_flex=True, sort_flex=True) ftruth2 = utils.open_flex(args.referenceForCCano) # for comparing to CC ano
sad_index_params.indexing.stills.rmsd_min_px = 8 sad_index_params.indexing.debug = True sad_index_params.indexing.fft1d.characteristic_grid = 0.029 #sad_index_params.indexing.refinement_protocol.mode = "" # index two color pattern using fft1d orient = indexer_base.from_parameters( reflections=count_spots.as_single_shot_reflections(refls_strong, inplace=False), imagesets=[iset], params=sad_index_params) try: orient.index() except (Sorry, RuntimeError): print("\n\n\t INDEXING FAILED\n") failed_shots.append(idx) np.savetxt(failed_idx_f, failed_shots, fmt="%d") continue indexed_shots.append(idx) np.savetxt(indexed_f, indexed_shots, fmt="%d") exp_name = os.path.join(outdir, "exp_%d.json" % Nprocessed) refl_name = os.path.join(outdir, "refl_%d.pkl" % Nprocessed) orient.export_as_json(orient.refined_experiments, file_name=exp_name) utils.save_flex(orient.refined_reflections, refl_name) crystals[idx] = orient.refined_experiments.crystals()[0] Nprocessed += 1 utils.save_flex(crystals, os.path.join(outdir, "ssirp_cryst_r102.pkl"))
eB = Experiment() eB.beam = beamB eB.detector = detector eB.imageset = iset eB.crystal = cryst_model EL = ExperimentList() EL.append(eA) EL.append(eB) exp_json = os.path.join(out_dir, "expAB_%d.json" % idx) reflA_pkl = os.path.join(out_dir, "reflA_%d.pkl" % idx) reflB_pkl = os.path.join(out_dir, "reflB_%d.pkl" % idx) orientAB.export_as_json(EL, exp_json) utils.save_flex(reflsA, reflA_pkl) utils.save_flex(reflsB, reflB_pkl) dump_pkl = os.path.join(out_dir, "dump_%d.pkl" % idx) t = loader.times[idx] # event time sec, nsec, fid = t.seconds(), t.nanoseconds(), t.fiducial() t_num, _ = utils.make_event_time(sec, nsec, fid) dump = { "crystalAB": crystalAB, "event_time": t_num, "tsec": sec, "tnsec": nsec, "tfid": fid, #"res_opt": res_opt, #"color_opt": color_opt,
beam=BeamA, mask=mask) dump = utils.open_flex("dump_%d.pkl" % shot_idx) refls = dump['refls_strong'] spot_utils.as_single_shot_reflections(refls) iset = dblock.extract_imagesets()[0] if index: orientAB = indexer_two_color(reflections=refls, imagesets=[iset], params=mad_index_params) orientAB.index() C = orientAB.refined_experiments.crystals()[0] utils.save_flex({"C": C}, "C%d.pkl" % shot_idx) from dxtbx.model.experiment_list import Experiment, ExperimentList e = Experiment() e.detector = D e.beam = B e.crystal = C e.imageset = iset el = ExperimentList() el.append(e) orientAB.export_as_json(el, "exp%d.json" % shot_idx) ############## ############# ############## ############## ##############
'refl': refl, 'optX': optX, 'optY': optY, 'optCrystal': optCrystal, 'hit_idx': hit_i, 'fcalc_f': fcalc_f, 'idx_proc_file': idx_proc_file } output_dir = os.path.dirname(output_prefix) output_base = os.path.basename(output_prefix) output_file = os.path.join(output_dir, "good_hit_%d_%s" % (hit_i, output_base)) output_pkl = output_file + ".pkl" utils.save_flex(output_dump, output_pkl) if fine_scan: results, fineX, fineY, fineA = fine_xyrefine( output_pkl, n_jobs=n_jobs, use_weights=use_weights, raw_image=raw_img) #optA_fine = fineX * fineY * optX * optY * sqr(crystal.get_U()) * sqr(crystal.get_B()) optCrystal_fine = deepcopy(crystal) optCrystal_fine.set_A(fineA) output_dump['fine_scan_results'] = results output_dump['optX_fine'] = fineX output_dump['optY_fine'] = fineY
def run_paramList(Ntrials, odir, tag, rank, n_jobs, pkl_file): import os import sys from copy import deepcopy import numpy as np import h5py from IPython import embed from scipy.ndimage.morphology import binary_dilation import scitbx from scitbx.array_family import flex from scitbx.matrix import sqr,col from simtbx.nanoBragg import shapetype from simtbx.nanoBragg import nanoBragg import libtbx.load_env # possibly implicit from libtbx.development.timers import Profiler from dxtbx.model.crystal import CrystalFactory from cctbx import crystal,crystal_orientation from cxid9114 import utils from cxid9114.sim import sim_utils from cxid9114.spots import spot_utils from cxid9114.bigsim.bigsim_geom import DET,BEAM from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW from cxid9114.refine.jitter_refine import make_param_list from cxid9114.bigsim import sim_spectra from LS49.sim.step4_pad import microcrystal data_pack = utils.open_flex(pkl_file) CRYST = data_pack['crystalAB'] mos_spread_deg=0.015 mos_doms=1000 beam_size_mm=0.001 exposure_s=1 use_microcrystal=True Deff_A = 2200 length_um = 2.2 timelog = False crystal = microcrystal(Deff_A = Deff_A, length_um = length_um, beam_diameter_um = beam_size_mm*1000, verbose=False) spec_file = h5py.File("simMe_data_run62.h5", "r") spec_data = spec_file["hist_spec"] Umat_data = spec_file["Umats"] en_chans = spec_file["energy_bins"][()] ilow = np.abs(en_chans - ENERGY_LOW).argmin() ihigh = np.abs(en_chans - ENERGY_HIGH).argmin() wave_chans = ENERGY_CONV/en_chans sfall_main = sim_spectra.load_spectra("test_sfall.h5") refls_strong = data_pack['refls_strong'] strong_mask_img = spot_utils.strong_spot_mask( refls_strong, (1800,1800) ) # edge detection in the ground truth strong mask image reference_img = (binary_dilation(strong_mask_img, iterations=1).astype(int) - strong_mask_img.astype(int) ).astype(bool) param_fileout = os.path.join( odir, "rank%d_%s.pkl" % (rank, tag)) param_list = make_param_list( CRYST, DET, BEAM, Ntrials, rot=0.09, cell=0.1, eq=(1,1,0), min_Ncell=20, max_Ncell=40, min_mos_spread=0.005, max_mos_spread=0.02) for p in param_list: print(p['crystal'].get_unit_cell().parameters()) shot_idx = int(data_pack['img_f'].split("_")[-1].split(".")[0]) Fluxes = spec_data[2] #shot_idx] Pmax = param_list[0] F1max = 0 for i_trial in range(Ntrials): print ("<><><><><><><><><><><><><><>") print ("Job %d; Trial %d / %d" % (rank, i_trial+1, Ntrials)) print ("<><><><><><><><><><><><><><>") if (rank==0 and i_trial % smi_stride==0): print("GPU status") os.system("nvidia-smi") print("\n\n") print("CPU memory usage") mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'""" os.system(mem_usg) assert (len(wave_chans)==len(Fluxes)==len(sfall_main)) if np.sum(Fluxes)==0: print ("Cannot simulate with an all-zeros spectrum!") sys.exit() N = crystal.number_of_cells(sfall_main[0].unit_cell()) Ncells_abc = (N,N,N) if force_twocolor: Fluxes *= 0 Fluxes[ilow] = 1e12 Fluxes[ihigh]=1e12 P = param_list[i_trial] simsAB = sim_utils.sim_twocolors2( P['crystal'], DET, BEAM, sfall_main, en_chans, Fluxes, pids = None, profile="gauss", oversample=0, Ncells_abc = Ncells_abc, #P['Ncells_abc'], mos_dom=mos_doms, verbose=verbose, mos_spread=mos_spread_deg, #@mos_spread=P['mos_spread'], cuda=True, device_Id=rank, beamsize_mm=beamsize_mm, exposure_s=exposure_s, boost=crystal.domains_per_crystal) out = np.sum( [ simsAB[i][0] for i in simsAB.keys() if simsAB[i]], axis=0) if out.shape==(): print("This simsAB output has an empty shape, something is wrong!") sys.exit() trial_refls = spot_utils.refls_from_sims([out], DET, BEAM, thresh=thresh) trial_spotmask = spot_utils.strong_spot_mask( trial_refls, (1800,1800) ) trial_img = (binary_dilation(trial_spotmask, iterations=1).astype(int) - trial_spotmask.astype(int) ).astype(bool) comp_img = trial_img.astype(int) + reference_img.astype(int)*2 Nfalse_neg = (comp_img==2).sum() # reference has signal but trial doesnt Nfalse_pos = (comp_img==1).sum() # trial has signal but reference doesnt Ntrue_pos = (comp_img==3).sum() # Precision = float(Ntrue_pos) / (Ntrue_pos + Nfalse_pos) Recall = float(Ntrue_pos) / (Ntrue_pos + Nfalse_neg) F1 = 2.*(Precision*Recall) / (Precision+Recall) if F1 > F1max: Pmax = {'crystal': P['crystal'], 'mos_spread': P['mos_spread'], 'Ncells_abc': P['Ncells_abc'], "F1": F1} F1max = F1 print("Rank %d, Trial %d: F1score = %.5f" % (rank, i_trial, F1)) utils.save_flex(Pmax, param_fileout) return F1max
bad_idx = u_all_Hi.index((0, 0, 0)) u_all_Hi.pop(bad_idx) u_all_amp.pop(bad_idx) indices = flex.miller_index(u_all_Hi) data = flex.double(u_all_amp) # save as an MTZ mset = miller.set(symm, indices=indices, anomalous_flag=True) marray = miller.array(mset, data).set_observation_type_xray_amplitude() if args.mtzoutput: mtz = marray.as_mtz_dataset(column_root_label='fobs', wavelength=parameters.WAVELEN_HIGH) ob = mtz.mtz_object() ob.write(args.out) # save as a PKL else: utils.save_flex(marray, args.out) if args.truth is not None: Ftruth = utils.open_flex(args.truth) marray_2 = marray.select( marray.resolution_filter_selection(d_max=30, d_min=args.dhighres)) r, c = utils.compute_r_factor(Ftruth, marray_2, is_flex=True, sort_flex=True) print("Truth R-factor=%4f and CC-Delta-anom=%.4f out to %.4f Angstrom" % (r, c, args.dhighres))
iset = loader.get_imageset(loader.get_image_file()) dblock = DataBlockFactory.from_imageset(iset[idx:idx + 1])[0] refl = flex.reflection_table.from_observations(dblock, find_spot_params) imgs.append(img) refls.append(refl) info_fname = image_fname.replace(".h5", ".pkl") sim_data = utils.open_flex(info_fname) orient = indexer_two_color( reflections=count_spots.as_single_shot_reflections(refl, inplace=False), imagesets=[iset], params=indexing_params) try: orient.index() crystals = [o.crystal for o in orient.refined_experiments] rmsd = orient.best_rmsd sim_data["sim_indexed_rmsd"] = rmsd sim_data["sim_indexed_crystals"] = crystals sim_data["sim_indexed_refls"] = refl except: print("FAILED!") utils.save_flex(sim_data, info_fname + out_tag) images_and_refls_to_simview(output_pref, imgs, refls)
if np.any(kept_sel): kept_refls = reflsPP[pid].select(flex.bool(kept_sel)) spot_masks = spot_utils.strong_spot_mask(kept_refls, img_sh, as_composite=False) # color the kept reflections green! winner_patches = get_spot_patches(spot_masks, ec='C2') for p in winner_patches: ax.add_patch(p) ax.set_title("Panel %d; Some Winners!" % pid) # save the figure figdir = os.path.join(outdir, "panel%d" % pid) if not os.path.exists(figdir): os.makedirs(figdir) figname = "pan%d_trial%d.jpg" % (pid, Ntrial) figpath = os.path.join(figdir, figname) plt.savefig(figpath, dpi=50) while ax.patches: ax.patches.pop() dump_name = outputname.replace(".pkl", "_probable_refls.pkl") utils.save_flex(good_refls, dump_name) if save_sims: h5.close()
except: pass crystalAB = orientAB.refined_experiments.crystals()[0] E = Experiment() E.beam = BEAM E.detector = DET E.imageset = iset E.crystal = crystalAB EL = ExperimentList() EL.append(E) exp_json = os.path.join(out_dir, "exp_%d_%s.json" % (idx, tag) ) refl_pkl = os.path.join(out_dir, "refl_%d_%s.pkl" % (idx, tag)) orientAB.export_as_json( EL, exp_json) utils.save_flex(refls_strong, refl_pkl) t = loader.times[idx] # event time sec, nsec, fid = t.seconds(), t.nanoseconds(), t.fiducial() t_num, _ = utils.make_event_time( sec, nsec, fid) dump = {"crystalAB": crystalAB, "event_time": t_num, "tsec": sec,"tnsec": nsec, "tfid": fid, "beamA": beamA, "beamB": beamB, "detector": detector, "rmsd": orientAB.best_rmsd, "refls_strong": refls_strong} dump_pkl= os.path.join(out_dir, "dump_%d_%s.pkl" % (idx, tag))
a=ucell_a, b=ucell_b, c=ucell_c) # for the non-outlier reflections, store the overlap values for each trial # then we can do a global clustering to see if there is a preferred simulation parameter data_dict["scores"] = list(map(list, overlaps[winners].T)) Amats = [list(C.get_A()) for C in new_crystals] data_dict["Amat"] = Amats df = pandas.DataFrame(data_dict) df['ave_score'] = np.vstack(df.scores.values).mean(1) df.to_pickle(outputname) idx_max = df.ave_score.idxmax() best_model = df.iloc[idx_max].drop(['scores', "Amat"]) best_Amat = tuple(df.iloc[idx_max]['Amat']) with open(outputname.replace(".pkl", "_best.txt"), "w") as o: o.write(best_model.to_string()) dump_name = outputname.replace(".pkl", "_best_dump.pkl") best_cryst = deepcopy(new_crystals[0]) best_cryst.set_A(best_Amat) utils.save_flex({ "good_refls": good_refls, "best_cryst": best_cryst }, dump_name)
'direction': (0.0, 0.0, 1.0), 'divergence': 0.0, 'flux': 1e12, 'polarization_fraction': 1., 'polarization_normal': (0.0, 1.0, 0.0), 'sigma_divergence': 0.0, 'transmission': 1.0, 'wavelength': 1.4 } # dxtbx crystal description hall_sym = sgtbx.space_group_info( number=19).type().hall_symbol() # ' P 2ac 2ab' cryst_descr = { '__id__': 'crystal', 'real_space_a': (127.7, 0, 0), 'real_space_b': (0, 225.4, 0), 'real_space_c': (0, 0, 306.1), 'space_group_hall_symbol': hall_sym } beam = BeamFactory.from_dict(beam_descr) whole_det = DetectorFactory.from_dict(cspad) cryst = CrystalFactory.from_dict(cryst_descr) from cxid9114 import utils utils.save_flex(whole_det, "ps2.det.pkl") utils.save_flex(cryst, "ps2.cryst.pkl") utils.save_flex(beam, "ps2.beam.pkl")
hkl_map2 = {v: k for k, v in hkl_map.iteritems()} Nhkl = len(hkl_map) assert (Nh == Nhkl) hout, Iout = [], [] for i in range(Nhkl): h = hkl_map2[i] val = IA[i] hout.append(h) Iout.append(np.exp(val)) sg = sgtbx.space_group(" P 4nw 2abw") Symm = crystal.symmetry(unit_cell=(79, 79, 38, 90, 90, 90), space_group=sg) hout = tuple(hout) mil_idx = flex.miller_index(hout) mil_set = miller.set(crystal_symmetry=Symm, indices=mil_idx, anomalous_flag=True) Iout_flex = flex.double(np.ascontiguousarray(Iout)) mil_ar = miller.array(mil_set, data=Iout_flex).set_observation_type_xray_intensity() utils.save_flex(mil_ar, "n00b_begin.pkl") waveA = ENERGY_CONV / 8944. waveA = ENERGY_CONV / 9034 IA = mil_ar out = IA.as_mtz_dataset(column_root_label="Iobs", title="B", wavelength=waveA) out.add_miller_array(miller_array=IA.average_bijvoet_mates(), column_root_label="IMEAN") obj = out.mtz_object() obj.write(args.o)
beamA, FF, ENERGIES, FLUX, pids=None, profile='tophat', # new_shape['shape'], oversample=0, # this should let nanoBragg decide how to oversample! Ncells_abc=new_shape['Ncells_abc'], mos_dom=Nmos_dom, mos_spread=0, #new_shape['mos_spread'], roi_pp=roi_pp, counts_pp=counts_pp, cuda=False) JFC = 10 sim_imgs = np.array(simsAB[0]) + np.array(simsAB[1]) h5.create_dataset("sim_imgs_%d" % i_trial, data=sim_imgs) print("\t... took %.4f seconds" % (time.time() - t)) Ncells_abc.append(new_shape['Ncells_abc']) mos_doms.append(Nmos_dom) mos_spread.append(new_shape['mos_spread']) xtal_shapes.append(new_shape['shape']) a, b, c, _, _, _ = new_crystal.get_unit_cell().parameters() ucell_a.append(a) ucell_b.append(b) ucell_c.append(c) new_crystals.append(new_crystal) utils.save_flex(refls_strong, outputname + "_refls.pkl")
Ncells_abc=(7, 7, 7), mos_dom=20, mos_spread=0.0) refl_simA_old = spot_utils.refls_from_sims(simsAB_old[0], detector, beamA) refl_simB_old = spot_utils.refls_from_sims(simsAB_old[1], detector, beamB) residA_old = metrics.check_indexable( refls_strong, refl_simA_old, detector, beamA, crystalAB, hkl_tol) residB_old = metrics.check_indexable( refls_strong, refl_simB_old, detector, beamB, crystalAB, hkl_tol) dump = {"crystalAB": crystalAB, "optCrystal": optCrystal, "residA": residA, "residB": residB, "residA_old": residA_old, "residB_old": residB_old, "beamA": beamA, "beamB": beamB, "overlap": overlap, "detector": detector, "refls_simA": refl_simA, "refls_simB": refl_simB, "refls_simA_old": refl_simA_old, "refls_simB_old": refl_simB_old, "rmsd_v1": data['rmsd'], "refls_strong": refls_strong} dump_name = data_name.replace(".pkl", "_%s.pkl" % tag) utils.save_flex(dump, dump_name) print "Wrote %s" % dump_name
data[h]["sigB"] = sigB can_analyze_spec = sigA > thresh or sigB > thresh if can_analyze_spec: if sigA < 0: sigA = 0.0001 if sigB < 0: sigB = 0.0001 fracA, fracB = sigA / (sigA + sigB), sigB / (sigA + sigB) data[h]["fracA"] = fracA data[h]["fracB"] = fracB data[h]['can_analyze'] = True print "Shot %d has 2 color fractions %f" % (h, fracA) else: data[h]['fracA'] = 0.5 # NOTE: not sure what to do here.. data[h]['fracB'] = 0.5 data[h]['can_analyze'] = False utils.save_flex(data, output_name) N_has_spec = np.sum([1 for k in data if data[k]['spectrum'] is not None]) N_weak_spec = np.sum([ 1 for k in data if not data[k]['can_analyze'] and data[k]['spectrum'] is not None ]) print("%d / %d hits had spectrum data!" % (N_has_spec, Ndata)) print("%d / %d hits had spectrum data but it was too weak!" % (N_weak_spec, N_has_spec)) print("Saved pickle %s. Done. " % output_name)