def from_observations(datablock, params=None):
'''
Construct a reflection table from observations.
:param datablock: The datablock
:param params: The input parameters
:return: The reflection table of observations
'''
from dials.algorithms.spot_finding.factory \
import SpotFinderFactory
from libtbx import Auto
if params is None:
from dials.command_line.find_spots import phil_scope
from dials.util.phil import parse
params = phil_scope.fetch(source=parse("")).extract()
if params.spotfinder.filter.min_spot_size is Auto:
detector = datablock.extract_imagesets()[0].get_detector()
if detector[0].get_type() == 'SENSOR_PAD':
# smaller default value for pixel array detectors
params.spotfinder.filter.min_spot_size = 3
else:
params.spotfinder.filter.min_spot_size = 6
logger.info('Setting spotfinder.filter.min_spot_size=%i' %
(params.spotfinder.filter.min_spot_size))
# Get the integrator from the input parameters
logger.info('Configuring spot finder from input parameters')
find_spots = SpotFinderFactory.from_parameters(datablock=datablock,
params=params)
# Find the spots
return find_spots(datablock)
def work(filename, cl=[]):
from dials.command_line.find_spots import phil_scope as params
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = params.command_line_argument_interpreter()
for cla in cl:
params = params.fetch(interp.process(cla))
datablock = DataBlockFactory.from_filenames([filename])[0]
reflections = flex.reflection_table.from_observations(datablock, params.extract())
detector = datablock.unique_detectors()[0]
beam = datablock.unique_beams()[0]
return analyse(reflections, detector, beam)
def work(filename, cl=[]):
from dials.command_line.find_spots import phil_scope as params
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = params.command_line_argument_interpreter()
for cla in cl:
params = params.fetch(interp.process(cla))
datablock = DataBlockFactory.from_filenames([filename])[0]
reflections = flex.reflection_table.from_observations(
datablock, params.extract())
detector = datablock.unique_detectors()[0]
beam = datablock.unique_beams()[0]
return analyse(reflections, detector, beam)
def make_phil(phil_file=None, show_phil=False):
if phil_file:
with open(phil_file, "r") as pf:
spf_phil = ip.parse(pf.read())
else:
spf_phil = ip.parse(spf_params_string)
new_phil = spf_scope.fetch(source=spf_phil)
spf_params = new_phil.extract()
if show_phil:
diff_phil = spf_scope.fetch_diff(source=spf_phil)
diff_phil.show()
return spf_params
def from_observations(experiments, params=None):
"""
Construct a reflection table from observations.
:param experiments: The experiments
:param params: The input parameters
:return: The reflection table of observations
"""
from dials.algorithms.spot_finding.factory import SpotFinderFactory
if params is None:
from dials.command_line.find_spots import phil_scope
from dials.util.phil import parse
params = phil_scope.fetch(source=parse("")).extract()
if params.spotfinder.filter.min_spot_size is libtbx.Auto:
detector = experiments[0].imageset.get_detector()
if detector[0].get_type() == "SENSOR_PAD":
# smaller default value for pixel array detectors
params.spotfinder.filter.min_spot_size = 3
else:
params.spotfinder.filter.min_spot_size = 6
logger.info(
"Setting spotfinder.filter.min_spot_size=%i",
params.spotfinder.filter.min_spot_size,
)
# Get the integrator from the input parameters
logger.info("Configuring spot finder from input parameters")
find_spots = SpotFinderFactory.from_parameters(
experiments=experiments, params=params
)
# Find the spots
return find_spots(experiments)
return data
skip_weak = True
skip_failed = True
skip_indexed = True
weak_shots_f = os.path.join(out_dir, "weak_shots.txt")
failed_idx_f = os.path.join(out_dir, "failed_shots.txt")
indexed_f = os.path.join(out_dir, "indexed_shots.txt")
weak_shots = load_tracker_f(weak_shots_f)
failed_shots = load_tracker_f(failed_idx_f)
indexed_shots = load_tracker_f(indexed_f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par_moder = deepcopy(spot_par)
spot_par.spotfinder.threshold.dispersion.global_threshold = 70.
spot_par.spotfinder.threshold.dispersion.gain = 28.
spot_par.spotfinder.threshold.dispersion.kernel_size = [4, 4]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 2.25
spot_par.spotfinder.threshold.dispersion.sigma_background = 6.
spot_par.spotfinder.filter.min_spot_size = 2
spot_par.spotfinder.force_2d = True
spot_par.spotfinder.lookup.mask = mask_file
#spot_par_moder.spotfinder.threshold.dispersion.global_threshold = 56.
#spot_par_moder.spotfinder.threshold.dispersion.gain = 28.
#spot_par_moder.spotfinder.threshold.dispersion.kernel_size = [1,1]
#spot_par_moder.spotfinder.threshold.dispersion.sigma_strong = 2.5
def msi(jid):
"""
msi stands for mad --> spot --> index
Here we load a data image that has crystal diffraction
then we spot peaks and index the crystal image
"""
#time.sleep(jid)
import sys
import numpy as np
import glob
import os
from copy import deepcopy
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dials.command_line.stills_process import phil_scope\
as indexer_phil_scope
from dials.algorithms.indexing.lattice_search import basis_vector_search_phil_scope
from dials.array_family import flex
import dxtbx
from dxtbx.datablock import DataBlockFactory
from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList, Experiment
from libtbx.utils import Sorry as Sorry
from libtbx.phil import parse
from cxi_xdr_xes.command_line.two_color_process import two_color_phil_scope
from cxi_xdr_xes.two_color import two_color_indexer
indexer_two_color = two_color_indexer.indexer_two_color
from cctbx import crystal
from cxid9114 import utils
from cxid9114 import parameters
indexer_phil_scope.adopt_scope(two_color_phil_scope)
indexer_phil_scope.adopt_scope(basis_vector_search_phil_scope)
mad_index_params = indexer_phil_scope.extract()
fnames = glob.glob(glob_str)
out_dir = os.path.join( args.out_dir , "job%d" % jid)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
refls_dir = "refls_%s" % tag
Els_dir = "Els_%s" % tag
dumps_dir = "dumps_%s" % tag
for o in [refls_dir, Els_dir, dumps_dir]:
sub_o = os.path.join(out_dir, o)
if not os.path.exists(sub_o):
os.makedirs(sub_o)
# track shots that indexed, or shots that
# had too few spots to index, so can change parameters and try again
def load_tracker_f(fname):
data = []
if os.path.exists(fname):
data = np.loadtxt(fname, str)
if data.size and not data.shape:
data = list(set(data[None].astype(int)))
else:
data = list(set(data.astype(int)))
return data
skip_weak = False
skip_failed = True #False
skip_indexed = False
weak_shots_f = os.path.join(out_dir, "weak_shots.txt")
failed_idx_f = os.path.join(out_dir, "failed_shots.txt")
indexed_f = os.path.join(out_dir, "indexed_shots.txt")
weak_shots = load_tracker_f(weak_shots_f)
failed_shots = load_tracker_f(failed_idx_f)
indexed_shots = load_tracker_f(indexed_f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# ------ spotting parameters
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par.spotfinder.threshold.dispersion.global_threshold = 40
spot_par.spotfinder.threshold.dispersion.gain = 28
spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 1
spot_par.spotfinder.threshold.dispersion.sigma_background = 6
spot_par.spotfinder.filter.min_spot_size = 3
spot_par.spotfinder.force_2d = True
# ------ indexing parameters
KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P1")
#KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P43212")
mad_index_params.refinement.parameterisation.beam.fix = "all"
mad_index_params.refinement.parameterisation.detector.fix = "all"
#mad_index_params.refinement.verbosity = 3
#mad_index_params.refinement.reflections.outlier.algorithm = "null"
mad_index_params.indexing.stills.refine_all_candidates = do_stills_refine
mad_index_params.indexing.optimise_initial_basis_vectors = do_basis_refine
#mad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False
mad_index_params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info()
mad_index_params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell()
mad_index_params.indexing.refinement_protocol.d_min_start = None
mad_index_params.indexing.debug = True
mad_index_params.indexing.real_space_grid_search.characteristic_grid = 0.02
mad_index_params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
mad_index_params.indexing.known_symmetry.relative_length_tolerance = 0.3
mad_index_params.indexing.stills.rmsd_min_px = 20000
mad_index_params.indexing.refinement_protocol.n_macro_cycles = 1
mad_index_params.indexing.multiple_lattice_search.max_lattices = 1
mad_index_params.indexing.basis_vector_combinations.max_refine = 10000000000
#mad_index_params.indexing.basis_vector_combinations.max_combinations = 150
#mad_index_params.indexing.stills.candidate_outlier_rejection = False
mad_index_params.indexing.refinement_protocol.mode = None #$"repredict_only"
mad_index_params.indexing.two_color.high_energy = parameters.ENERGY_HIGH
mad_index_params.indexing.two_color.low_energy = parameters.ENERGY_LOW
mad_index_params.indexing.two_color.avg_energy = parameters.ENERGY_LOW * .5 + parameters.ENERGY_HIGH * .5
#mad_index_params.indexing.two_color.spiral_method = (1., 100000) # 1000000)
mad_index_params.indexing.two_color.n_unique_v = 22
#mad_index_params.indexing.two_color.block_size = 25
#mad_index_params.indexing.two_color.filter_by_mag = (10,3)
# ------
N = len(fnames)
idx_split = np.array_split(np.arange(N), n_jobs)
n_idx = 0 # number indexed
for idx in idx_split[jid]:
if args.Nmax is not None:
if idx == args.Nmax:
sys.exit()
img_f = fnames[idx]
loader = dxtbx.load(img_f)
iset = loader.get_imageset(filenames=[loader.get_image_file()] )
DET = loader.get_detector()
BEAM = loader.get_beam()
El = ExperimentListFactory.from_imageset_and_crystal(iset, crystal=None)
E = El[0]
from cxid9114.geom.multi_panel import CSPAD_refined, CSPAD
El[0].detector = CSPAD_refined
if idx in weak_shots and skip_weak:
print("Skipping weak shots %d" % idx)
continue
if idx in failed_shots and skip_failed:
print("Skipping failed idx shots %d" % idx)
continue
if idx in indexed_shots and skip_indexed:
print("Skipping already idx shots %d" % idx)
continue
refls_strong = flex.reflection_table.from_observations(El, spot_par)
if len(refls_strong) < min_spot_per_pattern:
print("Not enough spots shot %d, continuing!" % idx)
weak_shots.append(idx)
try:
np.savetxt(weak_shots_f, weak_shots, fmt="%d")
except:
pass
continue
# ==================================
# 2 color indexer of 2 color pattern
# ==================================
try:
orientAB = indexer_two_color(
reflections=refls_strong,
experiments=El,
params=mad_index_params)
orientAB.index()
except (Sorry, RuntimeError, AssertionError) as error:
print("####\nIndexingFailed T_T \n####")
print (error)
failed_shots.append(idx)
try:
np.savetxt(failed_idx_f, failed_shots, fmt="%d")
except:
pass
continue
print("####\n *_* IndexingWORKED %d *_* \n####" % idx)
n_idx += 1
indexed_shots.append(idx)
try:
np.savetxt(indexed_f, indexed_shots, fmt="%d")
except:
pass
crystalAB = orientAB.refined_experiments.crystals()[0]
refl_pkl = os.path.join(out_dir,refls_dir, "refl_%d_%s.pkl" % (idx, tag))
utils.save_flex(refls_strong, refl_pkl)
El[0].crystal = crystalAB
El_json = os.path.join(out_dir, Els_dir, "El_%d_%s.json" % (idx, tag))
El.as_json(filename=El_json)
}
spotfinder {
filter {
max_spot_size = 1000
}
threshold {
algorithm = *dispersion dispersion_extended
dispersion {
gain = 1
global_threshold = 0
}
}
}
"""
spf_work_phil = ip.parse(custom_spf_string)
spf_phil = spf_scope.fetch(source=spf_work_phil)
# Custom PHIL for indexing
custom_idx_string = """
output {
experiments_filename = None
indexed_filename = None
refined_experiments_filename = None
integrated_experiments_filename = None
integrated_filename = None
profile_filename = None
integration_pickle = None
}
indexing {
refinement_protocol {
d_min_start = 2.0
def main(rank):
device_Id = rank % ngpu
worker_Id = node_id * ngpu + rank
import os
import sys
from copy import deepcopy
import glob
from itertools import izip
from scipy.spatial import distance
import h5py
import scipy.ndimage
from IPython import embed
import numpy as np
import pandas
from scipy.spatial import cKDTree
from simtbx.nanoBragg import shapetype, nanoBragg
from libtbx.phil import parse
from scitbx.matrix import sqr
import dxtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.crystal import CrystalFactory
from dials.algorithms.indexing.compare_orientation_matrices \
import rotation_matrix_differences
from dials.array_family import flex
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from cxid9114.refine import metrics
from cxid9114 import utils
from cxid9114.geom import geom_utils
from cxid9114.spots import integrate, spot_utils
from cxid9114 import parameters
from cxid9114.sim import sim_utils
from cctbx import miller, sgtbx
from cxid9114 import utils
from cxid9114.bigsim import sim_spectra
from cxid9114.refine.jitter_refine import make_param_list
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par.spotfinder.threshold.dispersion.global_threshold = 40
spot_par.spotfinder.threshold.dispersion.gain = 28
spot_par.spotfinder.threshold.dispersion.kernel_size = [2, 2]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 1
spot_par.spotfinder.threshold.dispersion.sigma_background = 6
spot_par.spotfinder.filter.min_spot_size = 3
spot_par.spotfinder.force_2d = True
odir = args.odir
odirj = os.path.join(odir, "job%d" % worker_Id)
#all_pkl_files = [s for sl in \
# [ files for _,_, files in os.walk(odir)]\
# for s in sl if s.endswith("pkl")]
#print "Found %d pkl files already in %s!" \
# % (len(all_pkl_files), odir)
if not os.path.exists(odirj):
os.makedirs(odirj)
hkl_tol = .15
run = 61
shot_idx = 0
ENERGIES = [parameters.ENERGY_LOW,
parameters.ENERGY_HIGH] # colors of the beams
FF = [10000, None]
cryst_descr = {
'__id__': 'crystal',
'real_space_a': (79, 0, 0),
'real_space_b': (0, 79, 0),
'real_space_c': (0, 0, 38),
'space_group_hall_symbol': '-P 4 2'
}
crystalAB = CrystalFactory.from_dict(cryst_descr)
sfall_main = sim_spectra.load_spectra("../bigsim/test_sfall.h5")
FFdat = [sfall_main[19], sfall_main[110]]
FLUX = [1e11, 1e11] # fluxes of the beams
chanA_flux = 1e11
chanB_flux = 1e11
FLUXdat = [chanA_flux, chanB_flux]
GAIN = 1
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
from cxid9114.bigsim.bigsim_geom import DET, BEAM
detector = DET
print("Rank %d Begin" % worker_Id)
for i_data in range(args.num_trials):
pklname = "%s_rank%d_data%d.pkl" % (ofile, worker_Id, i_data)
pklname = os.path.join(odirj, pklname)
print("<><><><><><><")
print("Job %d: trial %d / %d" %
(worker_Id, i_data + 1, args.num_trials))
print("<><><><><><><")
if (worker_Id == 0 and i_data % smi_stride == 0 and cuda):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg = """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
beamA = deepcopy(BEAM)
beamB = deepcopy(BEAM)
beamA.set_wavelength(waveA)
beamB.set_wavelength(waveB)
np.random.seed(args.seed)
crystalAB = CrystalFactory.from_dict(cryst_descr)
randnums = np.random.random(3)
Rrand = random_rotation(1, randnums)
crystalAB.set_U(Rrand.ravel())
#pert = np.random.uniform(0.0001/2/np.pi, 0.0003 / 2. /np.pi)
#print("PERT %f" % pert)
#Rsmall = random_rotation(0.00001, randnums ) #pert)
params_lst = make_param_list(crystalAB,
DET,
BEAM,
1,
rot=0.08,
cell=.0000001,
eq=(1, 1, 0),
min_Ncell=23,
max_Ncell=24,
min_mos_spread=0.02,
max_mos_spread=0.08)
Ctruth = params_lst[0]['crystal']
print Ctruth.get_unit_cell().parameters()
print crystalAB.get_unit_cell().parameters()
init_comp = rotation_matrix_differences((Ctruth, crystalAB))
init_rot = float(init_comp.split("\n")[-2].split()[2])
if use_data_spec:
print "NOT IMPLEMENTED, Using a phony 2col spectrum to simulate the data"
data_fluxes = FLUXdat
data_energies = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH]
data_ff = FFdat
else:
print "Using a phony two color spectrum to simulate the data"
data_fluxes = FLUXdat
data_energies = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH]
data_ff = FFdat
print("Truth crystal Misorientation deviation: %f deg" % init_rot)
if args.truth_cryst:
print "Using truth crystal"
dataCryst = Ctruth
else:
print "Not using truth crystal"
dataCryst = crystalAB
if not make_background:
print "SIMULATING Flat-Fhkl IMAGES"
simsAB = sim_utils.sim_twocolors2(
crystalAB,
detector,
BEAM,
FF, [parameters.ENERGY_LOW, parameters.ENERGY_HIGH],
FLUX,
pids=None,
Gauss=Gauss,
cuda=cuda,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mos_doms,
mos_spread=mos_spread,
exposure_s=exposure_s,
beamsize_mm=beamsize_mm,
device_Id=device_Id,
boost=boost)
if make_background:
print("MAKING BACKGROUND")
spec_file = h5py.File("../bigsim/simMe_data_run62.h5", "r")
ave_spec = np.mean(spec_file["hist_spec"][()], axis=0)
data_fluxes = [ave_spec[19], ave_spec[110]]
data_energies = spec_file["energy_bins"][()][[19, 110]]
data_ff = [1, 1] #*len(data_energies)
only_water = True
else:
only_water = False
print "SIULATING DATA IMAGE"
print data_fluxes
simsDataSum = sim_utils.sim_twocolors2(dataCryst,
detector,
BEAM,
data_ff,
data_energies,
data_fluxes,
pids=None,
Gauss=Gauss,
cuda=cuda,
oversample=oversample,
Ncells_abc=Ncells_abc,
accumulate=True,
mos_dom=mos_doms,
mos_spread=mos_spread,
boost=boost,
exposure_s=exposure_s,
beamsize_mm=beamsize_mm,
only_water=only_water,
device_Id=device_Id)
simsDataSum = np.array(simsDataSum)
if make_background:
bg_out = h5py.File(bg_name, "w")
bg_out.create_dataset("bigsim_d9114", data=simsDataSum[0])
print "Background made! Saved to file %s" % bg_name
sys.exit()
if add_background:
print("ADDING BG")
background = h5py.File(bg_name, "r")['bigsim_d9114'][()]
bg_scale = np.sum([39152412349.12075, 32315440627.406036])
bg_scale = np.sum(data_fluxes) / bg_scale
print "%.3e backgorund scale" % bg_scale
print "BG shape", background.shape
simsDataSum[0] += background * bg_scale
if add_noise:
print("ADDING NOISE")
for pidx in range(1):
SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=pidx)
SIM.exposure_s = exposure_s
SIM.beamsize_mm = beamsize_mm
SIM.flux = np.sum(data_fluxes)
SIM.detector_psf_kernel_radius_pixels = 5
SIM.detector_psf_type = shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm = 0
SIM.quantum_gain = 28
SIM.raw_pixels = flex.double(simsDataSum[pidx].ravel())
SIM.add_noise()
simsDataSum[pidx] = SIM.raw_pixels.as_numpy_array()\
.reshape(simsDataSum[0].shape)
SIM.free_all()
del SIM
if args.write_img:
print "SAVING DATAFILE"
h5name = "%s_rank%d_data%d.h5" % (ofile, worker_Id, i_data)
h5name = os.path.join(odirj, h5name)
fout = h5py.File(h5name, "w")
fout.create_dataset("bigsim_d9114", data=simsDataSum[0])
fout.create_dataset("crystalAB", data=crystalAB.get_A())
fout.create_dataset("dataCryst", data=dataCryst.get_A())
fout.close()
if args.write_sim_img:
print "SAVING DATAFILE"
for i_sim in simsAB:
sim_h5name = "%s_rank%d_sim%d_%d.h5" % (ofile, worker_Id,
i_data, i_sim)
sim_h5name = os.path.join(odirj, sim_h5name)
from IPython import embed
embed()
fout = h5py.File(sim_h5name, "w")
fout.create_dataset("bigsim_d9114", data=simsAB[i_sim][0])
fout.create_dataset("crystalAB", data=crystalAB.get_A())
fout.create_dataset("dataCryst", data=dataCryst.get_A())
fout.close()
print "RELFS FROM SIMS"
refl_simA = spot_utils.refls_from_sims(simsAB[0],
detector,
beamA,
thresh=thresh)
refl_simB = spot_utils.refls_from_sims(simsAB[1],
detector,
beamB,
thresh=thresh)
if use_dials_spotter:
print("DIALS SPOTTING")
El = utils.explist_from_numpyarrays(simsDataSum, DET, beamA)
refl_data = flex.reflection_table.from_observations(El, spot_par)
print("Found %d refls using DIALS spot finder" % len(refl_data))
else:
refl_data = spot_utils.refls_from_sims(simsDataSum, detector, beamA,\
thresh=thresh)
print("Found %d refls using threshold" % len(refl_data))
if len(refl_data) == 0:
print "Rank %d: No reflections found! " % (worker_Id)
continue
residA = metrics.check_indexable2(refl_data, refl_simA, detector,
beamA, crystalAB, hkl_tol)
residB = metrics.check_indexable2(refl_data, refl_simB, detector,
beamB, crystalAB, hkl_tol)
sg96 = sgtbx.space_group(" P 4nw 2abw")
FA = sfall_main[19] # utils.open_flex('SA.pkl') # ground truth values
FB = sfall_main[110] #utils.open_flex('SB.pkl') # ground truth values
HA = tuple([hkl for hkl in FA.indices()])
HB = tuple([hkl for hkl in FB.indices()])
HA_val_map = {h: data for h, data in izip(FA.indices(), FA.data())}
HB_val_map = {h: data for h, data in izip(FB.indices(), FB.data())}
def get_val_at_hkl(hkl, val_map):
poss_equivs = [
i.h() for i in miller.sym_equiv_indices(sg96, hkl).indices()
]
in_map = False
for hkl2 in poss_equivs:
if hkl2 in val_map: # fast lookup
in_map = True
break
if in_map:
return hkl2, val_map[hkl2]
else:
return (None, None, None), -1
filt = 1 #True #`False #True
if filt:
_, all_HiA, _ = spot_utils.refls_to_hkl(refl_simA,
detector,
beamA,
crystal=crystalAB,
returnQ=True)
all_treeA = cKDTree(all_HiA)
nnA = all_treeA.query_ball_point(all_HiA, r=1e-7)
_, all_HiB, _ = spot_utils.refls_to_hkl(refl_simB,
detector,
beamB,
crystal=crystalAB,
returnQ=True)
all_treeB = cKDTree(all_HiB)
nnB = all_treeB.query_ball_point(all_HiB, r=1e-7)
NreflA = len(refl_simA)
NreflB = len(refl_simB)
drop_meA = []
for i, vals in enumerate(nnA):
if i in drop_meA:
continue
if len(vals) > 1:
pids = [refl_simA[v]['panel'] for v in vals]
if len(set(pids)) == 1:
refl_vals = refl_simA.select(
flex.bool(
[i_v in vals for i_v in np.arange(NreflA)]))
x, y, z = spot_utils.xyz_from_refl(refl_vals)
allI = [r['intensity.sum.value'] for r in refl_vals]
allI = sum(allI)
xm = np.mean(x)
ym = np.mean(y)
zm = np.mean(z)
drop_meA.extend(vals[1:])
x1b, x2b, y1b, y2b, z1b, z2b = zip(
*[r['bbox'] for r in refl_vals])
keep_me = vals[0]
# indexing order is important to modify as reference
refl_simA['intensity.sum.value'][keep_me] = allI
refl_simA['xyzobs.px.value'][keep_me] = (xm, ym, zm)
refl_simA['bbox'][keep_me] = (min(x1b), max(x2b),\
min(y1b), max(y2b), min(z1b), max(z2b))
else:
drop_meA.append(vals)
print vals
if drop_meA:
keep_meA = np.array([i not in drop_meA for i in range(NreflA)])
refl_simA = refl_simA.select(flex.bool(keep_meA))
NreflA = len(refl_simA)
drop_meB = []
for i, vals in enumerate(nnB):
if i in drop_meB:
continue
if len(vals) > 1:
pids = [refl_simB[v]['panel'] for v in vals]
if len(set(pids)) == 1:
print vals
# merge_spots(vals)
refl_vals = refl_simB.select(
flex.bool(
[i_v in vals for i_v in np.arange(NreflB)]))
x, y, z = spot_utils.xyz_from_refl(refl_vals)
allI = [r['intensity.sum.value'] for r in refl_vals]
allI = sum(allI)
xm = np.mean(x)
ym = np.mean(y)
zm = np.mean(z)
drop_meB.extend(vals[1:])
x1b, x2b, y1b, y2b, z1b, z2b = zip(
*[r['bbox'] for r in refl_vals])
keep_me = vals[0]
refl_simB['intensity.sum.value'][keep_me] = allI
refl_simB['xyzobs.px.value'][keep_me] = (xm, ym, zm)
refl_simB['bbox'][keep_me] = (min(x1b), max(x2b), min(y1b),\
max(y2b), min(z1b), max(z2b))
else:
drop_meB.append(vals)
print vals
if drop_meB:
keep_meB = [i not in drop_meB for i in range(NreflB)]
refl_simB = refl_simB.select(flex.bool(keep_meB))
NreflB = len(refl_simB)
## remake the trees given the drops
_, all_HiA = spot_utils.refls_to_hkl(refl_simA,
detector,
beamA,
crystal=crystalAB,
returnQ=False)
all_treeA = cKDTree(all_HiA)
_, all_HiB = spot_utils.refls_to_hkl(refl_simB,
detector,
beamB,
crystal=crystalAB,
returnQ=False)
#all_treeB = cKDTree(all_HiB)
## CHECK if same HKL, indexed by both colors
# exists on multiple panels, and if so, delete...
nnAB = all_treeA.query_ball_point(all_HiB, r=1e-7)
drop_meA = []
drop_meB = []
for iB, iA_vals in enumerate(nnAB):
if len(iA_vals) > 0:
assert (len(iA_vals) == 1)
iA = iA_vals[0]
pidA = refl_simA[iA]['panel']
pidB = refl_simB[iB]['panel']
if pidA != pidB:
drop_meA.append(iA)
drop_meB.append(iB)
if drop_meA:
keep_meA = [i not in drop_meA for i in range(NreflA)]
refl_simA = refl_simA.select(flex.bool(keep_meA))
if drop_meB:
keep_meB = [i not in drop_meB for i in range(NreflB)]
refl_simB = refl_simB.select(flex.bool(keep_meB))
# ---- Done with edge case filters#
# reflections per panel
rpp = spot_utils.refls_by_panelname(refl_data)
rppA = spot_utils.refls_by_panelname(refl_simA)
rppB = spot_utils.refls_by_panelname(refl_simB)
DATA = {
"D": [],
"IA": [],
"IB": [],
"h2": [],
"k2": [],
"l2": [],
"h": [],
"k": [],
"l": [],
"PA": [],
"PB": [],
"FA": [],
"FB": [],
"iA": [],
"iB": [],
"Nstrong": [],
"pid": [],
"delta_pix": [],
"deltaX": [],
"deltaY": []
}
all_int_me = []
# now set up boundboxes and integrate
if tilt_plane_integration:
mask = np.ones(simsDataSum.shape).astype(np.bool)
print "Using tilt plane integration!"
else:
print "Not using tilt plane integration, just basic spot thresh integration "
for pid in rpp:
if tilt_plane_integration:
Is, Ibk, noise, pix_per = \
integrate.integrate3(
rpp[pid],
mask[pid],
simsDataSum[pid],
gain=28) #nom_gain)
R = rpp[pid]
if pid in rppA: # are there A-channel reflections on this panel
inA = True
RA = rppA[pid]
xA, yA, _ = spot_utils.xyz_from_refl(RA)
pointsA = np.array(zip(xA, yA))
HA, HiA, QA = spot_utils.refls_to_hkl(RA,
detector,
beamA,
crystal=crystalAB,
returnQ=True)
else:
inA = False
if pid in rppB: # are there B channel reflections on this channel
inB = True
RB = rppB[pid]
xB, yB, _ = spot_utils.xyz_from_refl(RB)
pointsB = np.array(zip(xB, yB))
HB, HiB, QB = spot_utils.refls_to_hkl(RB,
detector,
beamB,
crystal=crystalAB,
returnQ=True)
else:
inB = False
x, y, _ = spot_utils.xyz_from_refl(R)
x = np.array(x)
y = np.array(y)
panX, panY = detector[pid].get_image_size()
mergesA = []
mergesB = []
if inA and inB: # are there both A and B channel reflections ? If so, lets find out which ones have same hkl
# make tree structure for merging the spots
treeA = cKDTree(pointsA)
treeB = cKDTree(pointsB)
QA = geom_utils.res_on_panel(detector[pid], beamA)
QAmag = np.linalg.norm(QA, axis=2) * 2 * np.pi
detdist = detector[pid].get_distance()
pixsize = detector[pid].get_pixel_size()[0]
merge_me = []
for p in pointsA:
iix, iiy = int(p[0]), int(p[1])
q = QAmag[iiy, iix]
radA = detdist * np.tan(
2 * np.arcsin(q * waveA / 4 / np.pi)) / pixsize
radB = detdist * np.tan(
2 * np.arcsin(q * waveB / 4 / np.pi)) / pixsize
rmax = np.abs(radA - radB)
split_spot_pairs = treeB.query_ball_point(x=p, r=rmax + sz)
merge_me.append(split_spot_pairs)
#rmax = geom_utils.twocolor_deltapix(detector[pid], beamA, beamB)
#merge_me = treeA.query_ball_tree(treeB, r=rmax + sz)
for iA, iB in enumerate(merge_me):
if not iB:
continue
iB = iB[0]
# check that the miller indices are the same
if not all([i == j for i, j in zip(HiA[iA], HiB[iB])]):
continue
x1A, x2A, y1A, y2A, _, _ = RA[iA]['bbox'] # shoebox'].bbox
x1B, x2B, y1B, y2B, _, _ = RB[iB]['bbox'] # shoebox'].bbox
xlow = max([0, min((x1A, x1B)) - sz])
xhigh = min([panX, max((x2A, x2B)) + sz])
ylow = max([0, min((y1A, y1B)) - sz])
yhigh = min([panY, max((y2A, y2B)) + sz])
#if iA==79:
# embed()
# integrate me if I am in the bounding box!
int_me = np.where((xlow < x) & (x < xhigh) & (ylow < y)
& (y < yhigh))[0]
if not int_me.size:
continue
mergesA.append(iA)
mergesB.append(iB)
# integrate the spot, this will change depending on data or simulation
totalI = 0
totalCOM = 0
for ref_idx in int_me:
if tilt_plane_integration:
totalI += Is[ref_idx]
else:
totalI += rpp[pid][ref_idx]["intensity.sum.value"]
totalCOM += np.array(
rpp[pid][ref_idx]["xyzobs.px.value"])
totalCOM /= len(int_me)
PA = RA[iA]['intensity.sum.value']
PB = RB[iB]['intensity.sum.value']
# get the hkl structure factor, and the sym equiv hkl
(h, k, l) = HiA[iA] # NOTE: same for A and B channels
(h2, k2, l2), FA = get_val_at_hkl((h, k, l), HA_val_map)
_, FB = get_val_at_hkl(
(h, k, l),
HB_val_map) # NOTE: no need to return h2,k2,l2 twice
#if FB==-1 or FA==-1:
# continue
DATA['h'].append(h)
DATA['k'].append(k)
DATA['l'].append(l)
DATA['h2'].append(h2)
DATA['k2'].append(k2)
DATA['l2'].append(l2)
DATA['D'].append(totalI)
DATA['PA'].append(PA)
DATA['PB'].append(PB)
DATA['FA'].append(FA)
DATA['FB'].append(FB)
DATA['IA'].append(abs(FA)**2)
DATA['IB'].append(abs(FB)**2)
DATA['pid'].append(pid)
DATA["Nstrong"].append(int_me.size)
DATA["iA"].append(iA)
DATA["iB"].append(iB)
all_int_me.append(int_me)
# NOTE: stash the sim-data distance (COM to COM)
posA = RA[iA]['xyzobs.px.value']
posB = RB[iB]['xyzobs.px.value']
simCOM = np.mean([posA, posB], axis=0)
DATA["delta_pix"].append(
distance.euclidean(totalCOM[:2], simCOM[:2]))
DATA["deltaX"].append(totalCOM[0] - simCOM[0])
DATA["deltaY"].append(totalCOM[1] - simCOM[1])
if inA:
for iA, ref in enumerate(RA):
if iA in mergesA:
continue
x1A, x2A, y1A, y2A, _, _ = RA[iA][
'bbox'] # ['shoebox'].bbox
xlow = max((0, x1A - sz))
xhigh = min((panX, x2A + sz))
ylow = max((0, y1A - sz))
yhigh = min((panY, y2A + sz))
int_me = np.where((xlow < x) & (x < xhigh) & (ylow < y)
& (y < yhigh))[0]
if not int_me.size:
continue
totalI = 0
totalCOM = 0
for ref_idx in int_me:
if tilt_plane_integration:
totalI += Is[ref_idx]
else:
totalI += rpp[pid][ref_idx]["intensity.sum.value"]
totalCOM += np.array(
rpp[pid][ref_idx]["xyzobs.px.value"])
totalCOM /= len(int_me)
PA = RA[iA]['intensity.sum.value']
PB = 0 # crucial ;)
# get the hkl structure factor, and the sym equiv hkl
(h, k, l) = HiA[iA] # NOTE: same for A and B channels
(h2, k2, l2), FA = get_val_at_hkl((h, k, l), HA_val_map)
_, FB = get_val_at_hkl(
(h, k, l),
HB_val_map) # NOTE: no need to return h2,k2,l2 twice
#if FA==-1 or FB==-1:
# continue
DATA['h'].append(h)
DATA['k'].append(k)
DATA['l'].append(l)
DATA['h2'].append(h2)
DATA['k2'].append(k2)
DATA['l2'].append(l2)
DATA['D'].append(totalI)
DATA['PA'].append(PA)
DATA['PB'].append(PB)
DATA['FA'].append(FA)
DATA['FB'].append(FB)
DATA['IA'].append(abs(FA)**2)
DATA['IB'].append(abs(FB)**2)
DATA['pid'].append(pid)
DATA["Nstrong"].append(int_me.size)
DATA["iA"].append(iA)
DATA["iB"].append(np.nan)
all_int_me.append(int_me)
# NOTE: stash the sim-data distance (COM to COM)
simCOM = np.array(RA[iA]['xyzobs.px.value'])
DATA["delta_pix"].append(
distance.euclidean(totalCOM[:2], simCOM[:2]))
DATA["deltaX"].append(totalCOM[0] - simCOM[0])
DATA["deltaY"].append(totalCOM[1] - simCOM[1])
if inB:
for iB, ref in enumerate(RB):
if iB in mergesB:
continue
x1B, x2B, y1B, y2B, _, _ = RB[iB]['bbox'] # shoebox'].bbox
xlow = max((0, x1B - sz))
xhigh = min((panX, x2B + sz))
ylow = max((0, y1B - sz))
yhigh = min((panY, y2B + sz))
# subimg = simsDataSum[pid][ylow:yhigh, xlow:xhigh]
# bg = 0
int_me = np.where((xlow < x) & (x < xhigh) & (ylow < y)
& (y < yhigh))[0]
if not int_me.size:
continue
totalI = 0
totalCOM = 0
for ref_idx in int_me:
if tilt_plane_integration:
totalI += Is[ref_idx]
else:
totalI += rpp[pid][ref_idx]["intensity.sum.value"]
totalCOM += np.array(
rpp[pid][ref_idx]["xyzobs.px.value"])
totalCOM /= len(int_me)
PA = 0 # crucial ;)
PB = RB[iB]['intensity.sum.value']
# get the hkl structure factor, and the sym equiv hkl
(h, k, l) = HiB[iB] # NOTE: same for A and B channels
(h2, k2, l2), FB = get_val_at_hkl((h, k, l), HB_val_map)
_, FA = get_val_at_hkl(
(h, k, l),
HA_val_map) # NOTE: no need to return h2,k2,l2 twice
#if FA==-1 or FB==-1:
# continue
DATA['h'].append(h)
DATA['k'].append(k)
DATA['l'].append(l)
DATA['h2'].append(h2)
DATA['k2'].append(k2)
DATA['l2'].append(l2)
DATA['D'].append(totalI)
DATA['PA'].append(PA)
DATA['PB'].append(PB)
DATA['FA'].append(FA)
DATA['FB'].append(FB)
DATA['IA'].append(abs(FA)**2)
DATA['IB'].append(abs(FB)**2)
DATA['pid'].append(pid)
DATA["Nstrong"].append(int_me.size)
DATA["iA"].append(np.nan)
DATA["iB"].append(iB)
all_int_me.append(int_me)
# NOTE: stash the sim-data distance (COM to COM)
simCOM = np.array(RB[iB]['xyzobs.px.value'])
DATA["delta_pix"].append(
distance.euclidean(totalCOM[:2], simCOM[:2]))
DATA["deltaX"].append(totalCOM[0] - simCOM[0])
DATA["deltaY"].append(totalCOM[1] - simCOM[1])
df = pandas.DataFrame(DATA)
df["run"] = run
df["shot_idx"] = shot_idx
df['gain'] = GAIN
if use_data_spec:
print "Setting LA, LB as sums over flux regions A,B"
df['LA'] = data_fluxes[:75].sum()
df['LB'] = data_fluxes[75:].sum()
else:
print "Setting LA LB as data_fluxes"
df['LA'] = data_fluxes[0]
df["LB"] = data_fluxes[1]
df['K'] = FF[0]**2 * FLUX[0]
df["rhs"] = df.gain * (df.IA * df.LA * (df.PA / df.K) + df.IB * df.LB *
(df.PB / df.K))
df["lhs"] = df.D
#df['data_name'] = data_name
df['init_rot'] = init_rot
df.to_pickle(pklname)
print("PLOT")
if args.plot:
import pylab as plt
plt.plot(df.lhs, df.rhs, '.')
plt.show()
print("DonDonee")
def main(rank):
device_Id = rank % ngpu
worker_Id = node_id*ngpu + rank
import os
import sys
from copy import deepcopy
import glob
from itertools import izip
from scipy.spatial import distance
import h5py
import scipy.ndimage
from IPython import embed
import numpy as np
import pandas
from scipy.spatial import cKDTree
from simtbx.nanoBragg import shapetype, nanoBragg
from libtbx.phil import parse
from scitbx.matrix import sqr
import dxtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.crystal import CrystalFactory
from dials.algorithms.indexing.compare_orientation_matrices \
import rotation_matrix_differences
from dials.array_family import flex
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from cxid9114.refine import metrics
from cxid9114 import utils
from cxid9114.geom import geom_utils
from cxid9114.spots import integrate, spot_utils
from cxid9114 import parameters
from cxid9114.sim import sim_utils
from cctbx import miller, sgtbx
from cxid9114 import utils
from cxid9114.bigsim import sim_spectra
from cxid9114.refine.jitter_refine import make_param_list
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par.spotfinder.threshold.dispersion.global_threshold = 40
spot_par.spotfinder.threshold.dispersion.gain = GAIN
spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 1
spot_par.spotfinder.threshold.dispersion.sigma_background = 6
spot_par.spotfinder.filter.min_spot_size = 3
spot_par.spotfinder.force_2d = True
odir = args.odir
odirj = os.path.join(odir, "job%d" % worker_Id)
#all_pkl_files = [s for sl in \
# [ files for _,_, files in os.walk(odir)]\
# for s in sl if s.endswith("pkl")]
#print "Found %d pkl files already in %s!" \
# % (len(all_pkl_files), odir)
if not os.path.exists(odirj):
os.makedirs(odirj)
hkl_tol = .15
run = 61
shot_idx = 0
ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams
FF = [10000, None]
cryst_descr = {'__id__': 'crystal',
'real_space_a': (79, 0, 0),
'real_space_b': (0, 79, 0),
'real_space_c': (0, 0, 38),
'space_group_hall_symbol': '-P 4 2'}
crystalAB = CrystalFactory.from_dict(cryst_descr)
sfall_main = sim_spectra.load_spectra("../bigsim/test_sfall.h5")
FFdat = [sfall_main[19], sfall_main[110]]
FLUX = [1e11, 1e11] # fluxes of the beams
chanA_flux = np.random.uniform(1e11,1e12)
chanB_flux = np.random.uniform(1e11,1e12)
FLUXdat = [chanA_flux, chanB_flux]
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
from cxid9114.bigsim.bigsim_geom import DET,BEAM
detector = DET
print("Rank %d Begin" % worker_Id)
for i_data in range( args.num_trials):
pklname = "%s_rank%d_data%d.pkl" % (ofile, worker_Id, i_data)
pklname = os.path.join( odirj, pklname)
print("<><><><><><><")
print("Job %d: trial %d / %d" % ( worker_Id, i_data+1, args.num_trials ))
print("<><><><><><><")
if (worker_Id==0 and i_data % smi_stride==0 and cuda):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
beamA = deepcopy(BEAM)
beamB = deepcopy(BEAM)
beamA.set_wavelength(waveA)
beamB.set_wavelength(waveB)
SCALE = np.random.uniform(0.1,10)
np.random.seed(args.seed)
crystalAB = CrystalFactory.from_dict(cryst_descr)
randnums = np.random.random(3)
Rrand = random_rotation(1, randnums)
crystalAB.set_U(Rrand.ravel())
#pert = np.random.uniform(0.0001/2/np.pi, 0.0003 / 2. /np.pi)
#print("PERT %f" % pert)
#Rsmall = random_rotation(0.00001, randnums ) #pert)
params_lst = make_param_list(crystalAB, DET, BEAM,
1, rot=0.08, cell=.0000001, eq=(1,1,0),
min_Ncell=23, max_Ncell=24,
min_mos_spread=0.02,
max_mos_spread=0.08)
Ctruth = params_lst[0]['crystal']
print Ctruth.get_unit_cell().parameters()
print crystalAB.get_unit_cell().parameters()
init_comp = rotation_matrix_differences((Ctruth, crystalAB))
init_rot = float(init_comp.split("\n")[-2].split()[2])
if use_data_spec:
print "NOT IMPLEMENTED, Using a phony 2col spectrum to simulate the data"
data_fluxes = FLUXdat
data_energies = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH]
data_ff = FFdat
else:
print "Using a phony two color spectrum to simulate the data"
data_fluxes = FLUXdat
data_energies = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH]
data_ff = FFdat
print ("Truth crystal Misorientation deviation: %f deg" % init_rot )
if args.truth_cryst:
print "Using truth crystal"
dataCryst = Ctruth
else:
print "Not using truth crystal"
dataCryst = crystalAB
if not make_background:
print "SIMULATING Flat-Fhkl IMAGES"
simsAB = sim_utils.sim_twocolors2(
crystalAB, detector, BEAM, FF,
[parameters.ENERGY_LOW, parameters.ENERGY_HIGH],
FLUX, pids=None, Gauss=Gauss, cuda=cuda, oversample=oversample,
Ncells_abc=Ncells_abc, mos_dom=mos_doms, mos_spread=mos_spread,
exposure_s=exposure_s, beamsize_mm=beamsize_mm, device_Id=device_Id,
boost=boost)
if make_background:
print("MAKING BACKGROUND")
spec_file = h5py.File("../bigsim/simMe_data_run62.h5", "r")
ave_spec = np.mean( spec_file["hist_spec"][()], axis=0)
data_fluxes=[ave_spec[19], ave_spec[110] ]
data_energies = spec_file["energy_bins"][()][[19,110]]
data_ff = [1,1] #*len(data_energies)
only_water=True
else:
only_water=False
print "SIULATING DATA IMAGE"
print data_fluxes
simsDataSum = sim_utils.sim_twocolors2(
dataCryst, detector, BEAM, data_ff,
data_energies,
data_fluxes, pids=None, Gauss=Gauss, cuda=cuda,oversample=oversample,
Ncells_abc=Ncells_abc, accumulate=True, mos_dom=mos_doms,
mos_spread=mos_spread, boost=boost,
exposure_s=exposure_s, beamsize_mm=beamsize_mm,
only_water=only_water, device_Id=device_Id)
simsDataSum = SCALE*np.array(simsDataSum)
if make_background:
bg_out = h5py.File(bg_name, "w")
bg_out.create_dataset("bigsim_d9114",data=simsDataSum[0])
print "Background made! Saved to file %s" % bg_name
sys.exit()
if add_background:
print("ADDING BG")
background = h5py.File(bg_name, "r")['bigsim_d9114'][()]
bg_scale = np.sum([39152412349.12075, 32315440627.406036] )
bg_scale = np.sum(data_fluxes) / bg_scale
print "%.3e backgorund scale" % bg_scale
print "BG shape", background.shape
simsDataSum[0] += background * bg_scale
if add_noise:
print("ADDING NOISE")
for pidx in range(1):
SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=pidx)
SIM.exposure_s = exposure_s
SIM.beamsize_mm = beamsize_mm
SIM.flux = np.sum(data_fluxes)
SIM.detector_psf_kernel_radius_pixels=5;
#SIM.detector_psf_type=shapetype.Gauss
SIM.detector_psf_type=shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm=0
SIM.quantum_gain = GAIN
SIM.raw_pixels = flex.double(simsDataSum[pidx].ravel())
SIM.add_noise()
simsDataSum[pidx] = SIM.raw_pixels.as_numpy_array()\
.reshape(simsDataSum[0].shape)
SIM.free_all()
del SIM
if args.write_img:
print "SAVING DATAFILE"
h5name = "%s_rank%d_data%d.h5" % (ofile, worker_Id, i_data)
h5name = os.path.join( odirj, h5name)
fout = h5py.File(h5name,"w" )
fout.create_dataset("bigsim_d9114", data=simsDataSum[0])
fout.create_dataset("crystalAB", data=crystalAB.get_A() )
fout.create_dataset("dataCryst", data=dataCryst.get_A() )
fout.close()
if args.write_sim_img:
print "SAVING DATAFILE"
for i_sim in simsAB:
sim_h5name = "%s_rank%d_sim%d_%d.h5" % (ofile, worker_Id, i_data, i_sim)
sim_h5name = os.path.join( odirj, sim_h5name)
fout = h5py.File(sim_h5name,"w" )
fout.create_dataset("bigsim_d9114",
data=simsAB[i_sim][0])
fout.create_dataset("crystalAB", data=crystalAB.get_A() )
fout.create_dataset("dataCryst", data=dataCryst.get_A() )
fout.close()
print "RELFS FROM SIMS"
refl_simA = spot_utils.refls_from_sims(simsAB[0], detector, beamA, thresh=thresh)
refl_simB = spot_utils.refls_from_sims(simsAB[1], detector, beamB, thresh=thresh)
if use_dials_spotter:
print("DIALS SPOTTING")
El = utils.explist_from_numpyarrays(simsDataSum,DET,beamA)
refl_data = flex.reflection_table.from_observations(El, spot_par)
print("Found %d refls using DIALS spot finder" % len(refl_data))
else:
refl_data = spot_utils.refls_from_sims(simsDataSum, detector, beamA,\
thresh=thresh)
print ("Found %d refls using threshold" % len(refl_data))
if len(refl_data)==0:
print "Rank %d: No reflections found! " % (worker_Id)
continue
residA = metrics.check_indexable2(
refl_data, refl_simA, detector, beamA, crystalAB, hkl_tol)
residB = metrics.check_indexable2(
refl_data, refl_simB, detector, beamB, crystalAB, hkl_tol)
FA = sfall_main[19] # utils.open_flex('SA.pkl') # ground truth values
FB = sfall_main[110] #utils.open_flex('SB.pkl') # ground truth values
HA = tuple([hkl for hkl in FA.indices()])
HB = tuple([hkl for hkl in FB.indices()])
HA_val_map = { h:data for h,data in izip(FA.indices(), FA.data())}
HB_val_map = { h:data for h,data in izip(FB.indices(), FB.data())}
Hmaps = [HA_val_map, HB_val_map]
def get_val_at_hkl(hkl, val_map):
sg96 = sgtbx.space_group(" P 4nw 2abw")
poss_equivs = [i.h() for i in
miller.sym_equiv_indices(sg96, hkl).indices()]
in_map=False
for hkl2 in poss_equivs:
if hkl2 in val_map: # fast lookup
in_map=True
break
if in_map:
return hkl2, val_map[hkl2]
else:
return (None,None,None),-1
if use_data_spec:
print "Setting LA, LB as sums over flux regions A,B"
LA = data_fluxes[:75].sum()
LB = data_fluxes[75:].sum()
else:
print "Setting LA LB as data_fluxes"
LA = data_fluxes[0]
LB = data_fluxes[1]
K=FF[0] ** 2 * FLUX[0]
L_at_color = [LA,LB]
out = spot_utils.integrate_boxes(
refl_data, simsDataSum[0], [refl_simA, refl_simB], DET,
[beamA,beamB], crystalAB, delta_q=args.deltaq, gain=GAIN )
Nh = len(out[0])
rhs = []
lhs = []
all_H2 = []
all_PA = []
all_PB = []
all_FA = []
all_FB = []
for i in range(Nh):
HKL = out[0][i]
yobs = out[1][i]
Pvals = out[2][i]
ycalc = 0
for i_P, P in enumerate(Pvals):
L = L_at_color[i_P]
H2, F = get_val_at_hkl(HKL, Hmaps[i_P])
if i_P==0:
all_FA.append(F)
else:
all_FB.append(F)
ycalc += SCALE*L*P*abs(F)**2/K
all_PA.append( Pvals[0])
all_PB.append( Pvals[1])
all_H2.append(H2)
rhs.append(ycalc)
lhs.append(yobs)
df = pandas.DataFrame({"rhs":rhs, "lhs": lhs,
"PA":all_PA, "PB":all_PB, "FA": all_FA,
"FB": all_FB})
df["run"] = run
df["shot_idx"] = shot_idx
df['gain'] = SCALE
df['LA'] = LA
df['LB'] = LB
df['K'] = K
df['init_rot'] = init_rot
h,k,l = zip(*all_H2)
df['h2'] = h
df['k2'] = k
df['l2'] = l
df.to_pickle(pklname)
if args.plot:
print("PLOT")
import pylab as plt
plt.plot( df.lhs, df.rhs, '.')
plt.show()
print("DonDonee")
