def __init__(self, config, detectorxbins=10, detectorybins=10, detectorzbins=10, infile=None): # If passed infile, will automatically read in the calibrated detector mean angles/sigmas DetectorResponse.__init__(self, config, detectorxbins, detectorybins, detectorzbins) self.means = np.zeros((3, self.npmt_bins)) self.sigmas = np.zeros(self.npmt_bins) if infile is not None: logger.info('Creating detector response / calibration with: %s' % infile) if infile.endswith('.h5'): try: self.read_from_hdf5(infile) except IOError as error: logger.warning( 'No calibration file found %s. Continuing uncalibrated' % infile) return if self.config.uuid != self.config_in_cal_file.uuid: logger.critical( 'UUID from calibration file does not match configuration: %s %s' % (self.config.uuid, self.config_in_cal_file.uuid)) exit(-1) # TODO: Raise an exception else: logger.info('Calibration file UUID matches') else: self.read_from_ROOT(infile) else: logger.warning( 'No calibration file specified. Continuing uncalibrated')
def print_tracks(tracks, count): logger.info('Total track count: %d' % len(tracks)) for index, track in enumerate(tracks): logger.info('Track %d: %s, %s, %f, norm: %f' % (index, str( track[0]), str(track[1]), track[2], np.linalg.norm(track[0]))) if index >= count: break
def save_configuration(self, config): global _config_list conf_name = config.config_name if conf_name in _config_list: logger.info('Replacing configuration: ' + conf_name) _config_list[conf_name] = config self._save_config_list() logger.info('Configuration saved: ' + conf_name)
def AVF_analyze_event(analyzer, event, debug=False): sig_cone = 0.01 lens_dia = None n_ph = 0 vtcs = analyzer.analyze_one_event_AVF(event, sig_cone, n_ph, min_tracks, chiC, temps, tol, debug, lens_dia) logger.info('Vertices: ' + str(vtcs)) return vtcs
def plot_tracks_from_endpoints(begin_pos, end_pos, pts=None, highlight_pt=None, path=None, show=True, skip_interval=50, plot_title="Tracks"): # Returns a 3D plot of tracks (a Tracks object), as lines extending from their # PMT hit position to the inscribed diameter of the detector. # If pts is not None, will also draw them (should be a (3,n) numpy array). # If highlight_pt exists, it will be colored differently. # If path exists, a path will be drawn between its points (should be shape (3,n)). hit_pos = end_pos[0::skip_interval].T source_pos = begin_pos[0::skip_interval].T logger.info('Plotting %d tracks' % len(hit_pos[0])) xs = np.vstack((hit_pos[0, :], source_pos[0, :])) ys = np.vstack((hit_pos[1, :], source_pos[1, :])) zs = np.vstack((hit_pos[2, :], source_pos[2, :])) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') # Draw track hit positions ax.scatter(hit_pos[0, :], hit_pos[1, :], hit_pos[2, :], color='red') # Draw tracks as lines for ii in range(len(hit_pos[0])): ax.plot(xs[:, ii], ys[:, ii], zs[:, ii], color='red') ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') plt.title(plot_title) # Draw pts if pts is not None: ax.scatter(pts[0, :], pts[1, :], pts[2, :], color='blue') # Draw highlight_pt, larger and different color if highlight_pt is not None: ax.scatter(highlight_pt[0], highlight_pt[1], highlight_pt[2], color='green', s=50) # Draw path between points in path if path is not None: ax.plot(path[0, :], path[1, :], path[2, :], color='blue') plt.title('Vertex position record') if show: plt.show() return fig
def __init__(self): global _config_list, _config_pickle_file if _config_list is None: try: with open(_configs_pickle_file, 'r') as f: _config_list = pickle.load(f) logger.info('Loaded config list: %s' % _configs_pickle_file) except IOError: _config_list = {} self._save_config_list()
def sph_scatter(sample_count, in_shell, out_shell): logger.info('sph_scatter shell radii: ' + str(in_shell) + ' ' + str(out_shell)) loc = np.random.uniform(-out_shell, out_shell, (sample_count, 3)) while len( loc[(np.linalg.norm(loc, axis=1) > in_shell) & (np.linalg.norm(loc, axis=1) <= out_shell)]) != sample_count: bl_idx = np.logical_not((np.linalg.norm(loc, axis=1) > in_shell) & (np.linalg.norm(loc, axis=1) <= out_shell)) smpl = sum(bl_idx) loc[bl_idx] = np.random.uniform(-out_shell, out_shell, (smpl, 3)) return loc
def assign_photons(npmt_bins, n_det, pmt_bins): start = time.time() pmt_photons = np.empty(npmt_bins, dtype=list) for photon in range(n_det): if photon % 1000000 == 0: logger.info("Photon " + str(photon) + " of " + str(n_det) + ': ' + str(time.time() - start)) pmt = int(pmt_bins[photon]) if pmt_photons[pmt] is None: pmt_photons[pmt] = [photon] else: pmt_photons[pmt] += [photon] return pmt_photons
def read_from_ROOT(self, filename): from ShortIO.root_short import GaussAngleRootReader # Read the means and sigmas from a ROOT file logger.info('Loading root calibration file: %s' % filename) self.is_calibrated = True reader = GaussAngleRootReader(filename) for bin_ind, mean, sigma in reader: self.means[:, bin_ind] = mean self.sigmas[bin_ind] = sigma if np.isnan(sigma): print "Nan read in for bin index " + str(bin_ind) logger.info('Last bin_index: %d' % bin_ind)
def _calibrate(config, photons_file, detresname, detxbins=10, detybins=10, detzbins=10, method="PDF", nevents=-1, datadir="", fast_calibration=True): logger.info('Calibrating with: ' + datadir + photons_file) if method == "PDF": dr = DetectorResponsePDF( config, detxbins, detybins, detzbins) # Do we need to continue to carry this? elif method == "GaussAngle": dr = DetectorResponseGaussAngle(config, detxbins, detybins, detzbins) else: logger.warning('Warning: using generic DetectorResponse base class.') dr = DetectorResponse(config) dr.calibrate(datadir + photons_file, datadir, nevents, fast_calibration=fast_calibration) logger.info( "=== Detector analysis calibration complete. Writing calibration file" ) if USE_ROOT: # In this case write both hdf5 and ROOT files dr.write_to_ROOT(datadir + detresname + '.root') # Config dict is just included for human readability (currently) detector_data = { 'config': dr.config, 'config_dict': vars(dr.config), 'means': dr.means, 'sigmas': dr.sigmas } dd.io.save(datadir + detresname + '.h5', detector_data)
def write(self, file_name): event = { 'track_tree': self.track_tree, 'gun': self.gun_specs, 'config_name': self.config_name, 'simulation_params': self.simulation_params } if self.config_name is not None: event['config'] = detectorconfig.get_detector_config( self.config_name) if self.photons is not None: event['photons'] = self.photons if self.full_event is not None: event['full_event'] = self.full_event event['tracks'] = self.tracks if hasattr(self.tracks, 'hit_pos'): event[ 'hit_pos'] = self.tracks.hit_pos # Note: these are the center of the lens that the photon hit event['means'] = self.tracks.means event['sigmas'] = self.tracks.sigmas logger.info('Writing deepdish file: ' + file_name) dd.io.save(file_name, event)
def load_from_file(cls, file_name): event = dd.io.load(file_name) config_name = event['config_name'] gun_specs = event['gun'] track_tree = event['track_tree'] tracks = event['tracks'] photons = event['photons'] simulation_params = event[ 'simulation_params'] if 'simulation_params' in event else None logger.info('Photon count: ' + str(len(photons))) event_file = cls(config_name, gun_specs, track_tree, tracks, photons, simulation_params=simulation_params) event_file.full_event = event['full_event'] # Preserve the whole thing in case we need access to 'hit_pos', 'means', 'sigmas' (for compatibility with the original HDF5 format) event_file.complete = event return event_file
def AVF_analyze_tracks(analyzer, tracks, debug=False): vtcs = analyzer.AVF(tracks, min_tracks, chiC, temps, tol, debug) logger.info('Vertices: ' + str(vtcs)) return vtcs
def load_or_build_detector(config, detector_material, g4_detector_parameters, force_build=False): configname = config.config_name filename_base = paths.detector_config_path + configname if not os.path.exists(paths.detector_config_path): os.makedirs(paths.detector_config_path) kabamland = None # How to ensure the material and detector parameters are correct?? if not force_build: try: detector_config = dd.io.load(filename_base + '.h5') kabamland = detector_config['detector'] logger.info("** Loaded HDF5 (deepdish) detector configuration: " + configname) except IOError as error: # Will dd throw an exception? try: with open(filename_base + '.pickle', 'rb') as pickle_file: kabamland = pickle.load(pickle_file) logger.info("** Loaded pickle detector configuration: " + configname) except IOError as error: pass if kabamland is not None: config_has_g4_dp = hasattr( kabamland, 'g4_detector_parameters' ) and kabamland.g4_detector_parameters is not None config_has_g4_dm = hasattr( kabamland, 'detector_material') and kabamland.detector_material is not None if g4_detector_parameters is not None: logger.info('*** Using Geant4 detector parameters specified' + (' - replacement' if config_has_g4_dp else '') + ' ***') kabamland.g4_detector_parameters = g4_detector_parameters elif config_has_g4_dp: logger.info( '*** Using Geant4 detector parameters found in loaded file ***' ) else: logger.info('*** No Geant4 detector parameters found at all ***') if detector_material is not None: logger.info('*** Using Geant4 detector material specified' + (' - replacement' if config_has_g4_dm else '') + ' ***') kabamland.detector_material = detector_material elif config_has_g4_dm: logger.info( '*** Using Geant4 detector material found in loaded file ***' ) else: logger.info('*** No Geant4 detector material found at all ***') else: from chroma.loader import load_bvh # Requires CUDA so only import it when necessary logger.info("** Building detector configuration: " + configname) kabamland = Detector(lm.create_scintillation_material(), g4_detector_parameters=g4_detector_parameters) kbl2.build_kabamland(kabamland, config) # view(kabamland) kabamland.flatten() kabamland.bvh = load_bvh(kabamland, bvh_name=config.config_name, read_bvh_cache=(not force_build)) ''' try: with open(filename_base+'.pickle','wb') as pickle_file: pickle.dump(kabamland, pickle_file) except IOError as error: logger.info("Error writing pickle file: " + filename_base+'.pickle') ''' # Write h5 file with configuration data structure logger.info('Saving h5 detector configuration. UUID: %s' % config.uuid) ''' # This was created to minimize what is saved from the Detector object. But for simplicity, we are currently pickling the whole object. detector_dict = { 'detector_material' : kabamland.detector_material, 'solids' : kabamland.solids, 'solid_rotations' : kabamland.solid_rotations, 'solid_displacements' : kabamland.solid_displacements, 'bvh' : kabamland.bvh, 'g4_detector_parameters' : kabamland.g4_detector_parameters, 'solid_id_to_channel_index' : kabamland.solid_id_to_channel_index, 'channel_index_to_solid_id' : kabamland.channel_index_to_solid_id, 'channel_index_to_channel_id' : kabamland.channel_index_to_channel_id, 'channel_id_to_channel_index' : kabamland.channel_id_to_channel_index, 'time_cdf' : kabamland.time_cdf, 'charge_cdf' : kabamland.charge_cdf } ''' # TODO: Saving the whole dict and the object is redundant # TODO: Also, saving all of kabamland vs. just the parameters above adds about 1 Meg to the file size (I think) import lenssystem ld_name = configname.split('_')[0][2:] lens_design = lenssystem.get_lens_sys(ld_name) config_data = { 'detector_config': config, 'detector_config_dict': vars(config), 'lens_config_dict': vars(lens_design) } detector_data = {'config': config_data, 'detector': kabamland} dd.io.save(filename_base + '.h5', detector_data) return kabamland
def simulate_and_calibrate(config, build_only=False, force=False, fast_calibration=True): config_name = config.config_name if (not force) and os.path.isfile( paths.get_calibration_file_name(config_name)): logger.info('Found calibration file: %s' % paths.get_calibration_file_name(config_name)) else: logger.info('Failed to find calibration file: ' + paths.get_calibration_file_name(config_name)) logger.info('==== Step 1: Setting up the detector ====') photons_file_base = 'sim-' + config_name + '_100million' photons_file_full_path_base = paths.detector_calibration_path + photons_file_base if force or ( not (os.path.exists(photons_file_full_path_base + '.root') or os.path.exists(photons_file_full_path_base + '.h5'))): if force: logger.info('Forcing detector build') logger.info('Starting to load/build: %s' % config_name) g4_detector_parameters = G4DetectorParameters( orb_radius=7., world_material='G4_Galactic') kabamland = utilities.load_or_build_detector( config, lm.create_scintillation_material(), g4_detector_parameters=g4_detector_parameters, force_build=force) logger.warning('=== Detector was loaded/built') if build_only: return logger.info('==== Simulating photons: %s ====' % photons_file_base) _full_detector_simulation(config, kabamland, 100000, photons_file_base, datadir=paths.detector_calibration_path) logger.warning('==== Simulation complete') simulation_file = photons_file_base + '.h5' elif os.path.exists( photons_file_full_path_base + '.h5' ): # TODO: The double if's and constantly adding extensions needs to be reworked simulation_file = photons_file_base + '.h5' else: # Fall back to root simulation_file = photons_file_base + '.root' logger.warning('==== Found/created photons file: %s ====' % simulation_file) if not build_only: logger.info("==== Step 2: Calibrating ====") _calibrate( config, simulation_file, paths.get_calibration_file_name_base_without_path(config_name), method="GaussAngle", nevents=10000, datadir=paths.detector_calibration_path, fast_calibration=fast_calibration) #os.remove(photons_file) # Would need to remove both logger.warning( '==== Calibration complete: %s %s ====' % (config_name, 'fast' if fast_calibration else 'slow'))
def __init__(self, config, detectorxbins=10, detectorybins=10, detectorzbins=10): # TODO: Duplicates a lot of stuff in the config self.config = config # To enable saving configuration with the calibration file self.configname = config.config_name # Adding this for intermediate calibration file writing self.is_calibrated = False self.lns_rad = config.half_EPD / config.EPD_ratio self.detectorxbins = detectorxbins self.detectorybins = detectorybins self.detectorzbins = detectorzbins #self.edge_length, self.facecoords, self.direction, self.axis, self.angle, self.spin_angle = return_values(config.edge_length, config.base) self.pmtxbins = config.pmtxbins self.pmtybins = config.pmtybins self.n_lens_sys = config.lens_count # Number of lens systems per face - TODO: not true anymore self.detector_r = config.detector_r self.nsteps = config.ring_count #self.n_triangles_per_surf = int(2*self.nsteps*int((self.nsteps-2)/2.)) #self.n_pmts_per_surf = int(self.n_triangles_per_surf/2.) #if not self.detector_r: # self.npmt_bins = 20*self.pmtxbins*self.pmtybins #else: # self.npmt_bins = 20*self.n_lens_sys*self.n_pmts_per_surf # One curved detecting surf for each lens system self.diameter_ratio = config.diameter_ratio self.thickness_ratio = config.thickness_ratio ##changed self.focal_length = config.focal_length ##end changed #self.pmt_side_length = np.sqrt(3)*(3-np.sqrt(5))*self.focal_length self.inscribed_radius = config.detector_radius #self.rotation_matrices = self.build_rotation_matrices() #self.inverse_rotation_matrices = np.linalg.inv(self.rotation_matrices) #self.displacement_matrix = self.build_displacement_matrix() #self.inverse_rotated_displacement_matrix = self.build_inverse_rotated_displacement_matrix() #self.lens_inverse_rotated_displacement_matrix = self.build_lensplane_inverse_rotated_displacement_matrix() #new properties for curved surface detectors # Comment this out to allow access to old calibration files self.triangle_centers, self.n_triangles_per_surf, self.ring = get_curved_surf_triangle_centers( config.vtx, self.lns_rad, self.detector_r, self.focal_length, self.nsteps, config.base_pixels) self.triangle_centers_tree = spatial.cKDTree(self.triangle_centers) self.n_pmts_per_surf = int(self.n_triangles_per_surf / 2.) if not self.detector_r: self.npmt_bins = 20 * self.pmtxbins * self.pmtybins else: self.npmt_bins = self.n_lens_sys * self.n_pmts_per_surf # One curved detecting surf for each lens system # Comment this out to allow access to old calibration files self.lens_centers = get_lens_triangle_centers( config.vtx, self.lns_rad, config.diameter_ratio, config.thickness_ratio, config.half_EPD, config.blockers, blocker_thickness_ratio=config.blocker_thickness_ratio, light_confinement=config.light_confinement, focal_length=config.focal_length, lens_system_name=config.lens_system_name) self.lens_rad = config.half_EPD #self.calc1 = self.pmtxbins/self.pmt_side_length #self.calc2 = self.pmtxbins/2.0 #self.calc3 = 2*self.pmtybins/(np.sqrt(3)*self.pmt_side_length) #self.calc4 = self.pmtybins/3.0 #self.calc5 = self.pmtxbins*self.pmtybins self.c_rings = np.cumsum(self.ring) self.c_rings_rolled = np.roll(self.c_rings, 1) self.c_rings_rolled[0] = 0 logger.info('Detector rings: %s, cumulative pixels in rings: %s' % (str(self.ring), str(self.c_rings)))
# Driver for generating new hdf5/dd event files and diagnosing AVF algorithm if __name__ == '__main__': import DetectorResponseGaussAngle import EventAnalyzer parser = argparse.ArgumentParser() parser.add_argument('h5_file', help='Event HDF5 file') args = parser.parse_args() event = DIEventFile.load_from_file(args.h5_file) title = str(event.gun_specs['energy']) + ' MeV ' + str( event.gun_specs['particle']) vertices = None if event.tracks is not None: logger.info('Track count: ' + str(len(event.tracks))) #event.tracks.sigmas.fill(0.01) # TODO: Temporary hack because I think we forced 0.0001 into the tracks in the test file. Sigmas too small really screw up the machine!! print_tracks(event.tracks, 20) calibrated_simulation = True if event.simulation_params is not None and 'calibrated' in event.simulation_params: calibrated_simulation = event.simulation_params['calibrated'] logger.info( '=== Simulation event used calibrated detector: %s ===' % str(calibrated_simulation)) else: logger.info('=== No calibration flag in event file ===') logger.info('Photons in file: %d' % len(event.full_event.photons_end)) config = detectorconfig.get_detector_config(
def fire_g4_particles(sample_count, config, particle, energy, inner_radius, outer_radius, h5_file, location=None, momentum=None, di_file_base=None, qe=None): from chroma.generator import vertex config_name = config.config_name sim, analyzer = sim_setup(config, paths.get_calibration_file_name(config_name), useGeant4=True, geant4_processes=1, no_gpu=False) #analyzer.det_res.is_calibrated=False # Temporary to test AVF with actual photon angles vs. calibration angles logger.info('Configuration:\t%s' % config_name) logger.info('Particle:\t\t%s ' % particle) logger.info('Energy:\t\t%d' % energy) logger.info('Sim count:\t\t%d' % sample_count) logger.info('File:\t\t%s' % h5_file) if location is None: # Location is a flag loc_array = sph_scatter(sample_count, inner_radius * 1000, outer_radius * 1000) else: loc_array = [location] with h5py.File(h5_file, 'w') as f: first = True logger.info('Running locations:\t%d' % len(loc_array)) for i, lg in enumerate(loc_array): logger.info('Location:\t\t%s' % str(lg)) if location is None: gun = vertex.particle_gun([particle], vertex.constant(lg), vertex.isotropic(), vertex.flat( float(energy) * 0.999, float(energy) * 1.001)) else: gun = vertex.particle_gun( [particle], vertex.constant(lg), vertex.constant(momentum), vertex.constant(energy) ) # TODO: AWS seems to require: vertex.constant(np.array(momentum)) events = sim.simulate(gun, keep_photons_beg=True, keep_photons_end=True, run_daq=False, max_steps=100) for ev in events: # Note: There is really only ever one event because we enumerate loc_array above vert = ev.photons_beg.pos tracks = analyzer.generate_tracks(ev, qe=qe) write_h5_reverse_track_file_event(f, vert, tracks, first) first = False #vertices = utilities.AVF_analyze_event(analyzer, ev) #utilities.plot_vertices(ev.photons_beg.track_tree, 'AVF plot', reconstructed_vertices=vertices) if di_file_base is not None: gun_specs = build_gun_specs(particle, lg, None, energy) di_file = DIEventFile(config_name, gun_specs, ev.photons_beg.track_tree, tracks, ev.photons_beg, ev) di_file.write(di_file_base + '_' + str(i) + '.h5') logger.info('Photons detected:\t%s' % str(tracks.sigmas.shape[0])) logger.info('============')
def plot_vertices(track_tree, title, with_electrons=True, file_name=None, reconstructed_vertices=None, reconstructed_vertices2=None): particles = {} energies = {} for key, value in track_tree.iteritems(): if 'particle' in value: particle = value['particle'] if particle not in particles: particles[particle] = [] energies[particle] = [] particles[particle].append(value['position']) energies[particle].append(100. * value['energy']) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') for key, value in particles.iteritems(): if with_electrons or key != 'e-': the_array = np.array(value) #ax.plot(the_array[:,0], the_array[:,1], the_array[:,2], '.', markersize=5.0) ax.scatter(the_array[:, 0], the_array[:, 1], the_array[:, 2], marker='o', s=energies[key], label=key) #), markersize=5.0) if reconstructed_vertices is not None: vertex_positions = [] for v in reconstructed_vertices: logger.info('Vertex position: %s' % v.pos) vertex_positions.append(np.asarray(v.pos)) vp = np.asarray(vertex_positions) logger.info('AVF positions: ' + str(vp)) ax.scatter(vp[:, 0], vp[:, 1], vp[:, 2], marker=(6, 1, 0), s=100., color='gray', label='AVF') #), markersize=5.0) # Optionally plot reconstructed vertices as well if reconstructed_vertices2 is not None: vertex_positions = [] for v in reconstructed_vertices2: logger.info('Vertex position: %s' % v.pos) vertex_positions.append(np.asarray(v.pos)) vp = np.asarray(vertex_positions) logger.info('AVF 2 positions: ' + str(vp)) ax.scatter(vp[:, 0], vp[:, 1], vp[:, 2], marker=(6, 1, 0), s=100., color='black', label='AVF 2') #), markersize=5.0) ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') ax.set_title(title) plt.legend(loc=2) # See https://pythonspot.com/3d-scatterplot/ # See: http://fredborg-braedstrup.dk/blog/2014/10/10/saving-mpl-figures-using-pickle if file_name is not None: pickle.dump(fig, file(file_name, 'wb')) plt.show()
def _full_detector_simulation(config, kabamland, amount, simname, datadir=""): # simulates 1000*amount photons uniformly spread throughout a sphere whose radius is the inscribed radius of the icosahedron. # Note that viewing may crash if there are too many lenses. (try using configview) from chroma.sim import Simulation # Require CUDA, so only import when necessary config_name = config.config_name file_name_base = datadir + simname if USE_ROOT: from ShortIO.root_short import ShortRootWriter f = ShortRootWrite(file_name_base + '.root') # SHERLOCK: Have to set the seed because Sherlock compute machines blow up if we use chroma's algorithm!!!! # sim = Simulation(kabamland, geant4_processes=0, seed=65432) # For now, does not take advantage of multiple cores # TODO: use sim_setup()? sim = Simulation( kabamland, geant4_processes=0 ) # For now, does not take advantage of multiple cores # TODO: use sim_setup()? with h5py.File(file_name_base + '.h5', 'w') as h5_file: # Total photons will be LOOP_COUNT * EVENT_COUNT * amount LOOP_COUNT = 100 EVENT_COUNT = 10 # Setup to write the hdf5 file incrementally # Can't use deepdish as it seems to require a single write which takes up too much memory start_pos = h5_file.create_dataset('photons_start', shape=(LOOP_COUNT * EVENT_COUNT, amount, 3), dtype=np.float32, chunks=True) end_pos = h5_file.create_dataset('photons_stop', shape=(LOOP_COUNT * EVENT_COUNT, amount, 3), dtype=np.float32, chunks=True) photon_flags = h5_file.create_dataset('photon_flags', shape=( LOOP_COUNT * EVENT_COUNT, amount, ), dtype=np.uint32, chunks=True) # Store the UUID and name to enable matching of the configuration, calibration, and simulation files h5_file.attrs['config_name'] = config_name h5_file.attrs['config_UUID'] = str(config.uuid) process = psutil.Process(os.getpid()) logger.info('Memory size: %d MB' % (process.memory_info().rss // 1000000)) for j in range(LOOP_COUNT): logger.info('%d of %d event sets' % (j, LOOP_COUNT)) ev_index = 0 sim_events = [ kb.uniform_photons(config.detector_radius, amount) for i in range(EVENT_COUNT) ] for ev in sim.simulate(sim_events, keep_photons_beg=True, keep_photons_end=True, run_daq=False, max_steps=100): start_pos[(j * EVENT_COUNT) + ev_index] = ev.photons_beg.pos end_pos[(j * EVENT_COUNT) + ev_index] = ev.photons_end.pos photon_flags[(j * EVENT_COUNT) + ev_index] = ev.photons_end.flags ev_index += 1 if USE_ROOT: # In this case, write both hdf5 and ROOT simulation files f.write_event(ev) logger.info('Memory size: %d MB' % (process.memory_info().rss // 1000000)) if USE_ROOT: f.close()
def calibrate(self, simname, directory=".", nevents=-1, fast_calibration=False): # Use with a simulation file 'simname' to calibrate the detector # Creates a list of mean angles and their uncertainties (sigma for # a cone of unit length), one for each PMT # There are 1000 events in a typical simulation. # Uses all photons hitting a given PMT at once (better estimate of sigma, # but may run out of memory in some cases). # Will not calibrate PMTs with <n_min hits logger.info('Fast calibration: %s' % str(fast_calibration)) self.is_calibrated = True start_time = time.time() base_hits_file_name = self.configname + '-hits' pickle_name = base_hits_file_name + '.pickle' hit_file_exists = False n_min = 10 # Do not calibrate a PMT if <n_min photons hit it try: logger.info('Attempting to load pickle hits file: %s%s' % (directory, pickle_name)) # TODO: This generally won't do anything because we no longer write this file with open(directory + pickle_name, 'rb') as inf: pmt_hits = pickle.load(inf) logger.info('Hit map pickle file loaded: ' + pickle_name) pmt_bins = pmt_hits['pmt_bins'] end_direction_array = pmt_hits['end_direction_array'] n_det = len(pmt_bins) hit_file_exists = True except IOError as error: # Assume file not found logger.info('Hit map pickle file not found. Creating: ' + base_hits_file_name) using_h5 = False if simname.endswith('.h5'): using_h5 = True ev_file = dd.io.load(simname) # TODO: Check the UUID!! events_in_file = len(ev_file['photons_start']) else: from ShortIO.root_short import GaussAngleRootWriter, GaussAngleRootReader, ShortRootReader reader = ShortRootReader(simname) events_in_file = len(reader) logger.info('Loaded simulation file: %s' % simname) logger.info('Simulation event count: %d' % events_in_file) if nevents < 1: nevents = events_in_file max_storage = min( nevents * 1000000, 120000000 ) #600M is too much, 400M is OK (for np.float32; using 300M) end_direction_array = np.empty((max_storage, 3), dtype=np.float32) pmt_bins = np.empty(max_storage, dtype=np.int) n_det = 0 # Loop through events, store for each photon the index of the PMT it hit (pmt_bins) # and the direction pointing back to its origin (end_direction_array) loops = 0 event_source = ev_file['photons_start'] if using_h5 else reader for index, ev_proxy in enumerate( event_source): # Not sure if we can enumerate a reader???? # TODO: This needs to be cleaned up if (using_h5): photons_beg = Photons(ev_proxy, [], [], []) photons_end = Photons(ev_file['photons_stop'][index], [], [], [], flags=ev_file['photon_flags'][index]) else: photons_beg = ev_proxy.photons_beg photons_end = ev_proxy.photons_end loops += 1 if loops > nevents: break if loops % 100 == 0: logger.info("Event " + str(loops) + " of " + str(nevents)) logger.handlers[0].flush() detected = (photons_end.flags & (0x1 << 2)).astype(bool) ''' reflected_diffuse = (ev.photons_end.flags & (0x1 << 5)).astype(bool) reflected_specular = (ev.photons_end.flags & (0x1 << 6)).astype(bool) logger.info("Total detected: " + str(sum(detected * 1))) logger.info("Total reflected: " + str(sum(reflected_diffuse * 1) + sum(reflected_specular * 1))) good_photons = detected & np.logical_not(reflected_diffuse) & np.logical_not(reflected_specular) logger.info("Total detected and not reflected: " + str(sum(good_photons * 1))) ''' if fast_calibration: ending_photons, length = self._find_photons_for_pmt( photons_beg.pos, photons_end.pos, detected, end_direction_array, n_det, max_storage) pmt_b = self.find_pmt_bin_array(ending_photons) if length is None: break else: beginning_photons = photons_beg.pos[ detected] # Include reflected photons ending_photons = photons_end.pos[detected] length = np.shape(ending_photons)[0] pmt_b = self.find_pmt_bin_array(ending_photons) end_point = self.lens_centers[pmt_b / self.n_pmts_per_surf] end_dir = normalize(end_point - beginning_photons) # if end_direction_array is None: # end_direction_array = end_dir # else: # end_direction_array = np.vstack((end_direction_array, end_dir)) #end_direction_array.append(end_dir) if n_det + length > max_storage: logger.info( 'Too many photons to store in memory; not reading any further events.' ) break end_direction_array[n_det:(n_det + length), :] = end_dir # if pmt_bins is None: # pmt_bins = pmt_b # else: # pmt_bins = np.hstack((pmt_bins, pmt_b)) #pmt_bins.append(pmt_b) pmt_bins[n_det:(n_det + length)] = pmt_b n_det += length if loops % 100 == 0: logger.info('Photons detected so far: ' + str(n_det + length)) # logger.info('Sample pmt bins: ' + str(pmt_bins[n_det:(n_det+length)])) logger.info("Time: " + str(time.time() - start_time)) total_means = np.zeros((self.npmt_bins, 3)) total_variances = np.zeros((self.npmt_bins)) total_u_minus_v = np.zeros((self.npmt_bins)) amount_of_hits = np.zeros((self.npmt_bins)) end_direction_array.resize((n_det, 3)) logger.info("Time: " + str(time.time() - start_time)) pmt_bins.resize(n_det) if not hit_file_exists: # TODO: No longer writing the pickle file # Write the pickle hits file regardless of whether using fast_calibration or not #pmt_hits = {'pmt_bins': pmt_bins, 'end_direction_array': end_direction_array} #with open(directory+pickle_name, 'wb') as outf: # pickle.dump(pmt_hits, outf) with h5py.File(directory + base_hits_file_name + '.h5', 'w') as h5file: _ = h5file.create_dataset( 'pmt_bins', data=pmt_bins, chunks=True) # TODO: Should we assign max shape? _ = h5file.create_dataset( 'end_direction_array', data=end_direction_array, chunks=True) # TODO: Should we assign max shape? logger.info('Hit map file created: ' + pickle_name) logger.info( "Finished collecting photons (or loading photon hit list). Time: " + str(time.time() - start_time)) if fast_calibration: bins_base_file_name = self.configname + '-pmt-bins' bins_pickle_file = bins_base_file_name + '.pickle' try: with open( directory + bins_pickle_file, 'rb' ) as inf: # TODO: This generally won't do anything because we no longer write this file pmt_photons = pickle.load(inf) logger.info('PMT photon list pickle file loaded: ' + bins_pickle_file) except IOError as error: start_assign = time.time() pmt_photons = assign_photons(self.npmt_bins, n_det, pmt_bins) logger.info("assign_photons took: " + str(time.time() - start_assign)) # TODO: No longer writing the pickle file #with open(directory + bins_pickle_file, 'wb') as outf: # pickle.dump(pmt_photons, outf) # TODO: Pure H5 does not work. (Because?) #with h5py.File(directory + bins_file + '.h5', 'w') as h5file: # _ = h5file.create_dataset('photon_pmts', data=pmt_photons, chunks=True) # Should we assign max shape? #logger.info('Type: ' + str(type(pmt_photons)) + ' ' + str(type(pmt_photons[0]))) dd.io.save(directory + bins_base_file_name + '.h5', pmt_photons) logger.info('PMT photon list file created: ' + bins_base_file_name + '.h5') logger.info("Finished listing photons by pmt. Time: " + str(time.time() - start_time)) draw_pmt_ind = -1 # looping through each pmt in order to save a mean_angle and a variance for i in range(self.npmt_bins): if i % 10000 == 0: logger.info( str(i) + ' out of ' + str(self.npmt_bins) + ' PMTs') logger.handlers[0].flush() logger.info("Time: " + str(time.time() - start_time)) if fast_calibration: photon_list = pmt_photons[i] angles_for_pmt = end_direction_array[photon_list] n_angles = len(angles_for_pmt) # np.shape(angles_for_pmt)[0] # skipping pmts with <2 photon hits (in which case the variance will be undefined with ddof=1) # also skipping if <n_min photon hits if n_angles < 2 or n_angles < n_min: logger.warning('Not enough angles for PMT: %d, %d' % (i, n_angles)) continue mean_angle, variance, uvvar = compute_pmt_calibration( angles_for_pmt, n_min) else: pmt_indices = np.where(pmt_bins == i)[0] if np.shape(pmt_indices)[0] == 0: continue angles_for_pmt = end_direction_array[pmt_indices] n_angles = np.shape(angles_for_pmt)[0] #skipping pmts with <2 photon hits (in which case the variance will be undefined with ddof=1) #also skipping if <n_min photon hits if n_angles < 2 or n_angles < n_min: continue mean_angle = normalize(np.mean(angles_for_pmt, axis=0)) # For each PMT, get a pair of axes which form an # orthonormal coordinate system with the PMT mean direction u_dir = np.cross(mean_angle, np.array([0, 0, 1])) if not (np.dot(u_dir, u_dir) > 0): # In case mean_angle = [0,0,1] u_dir = np.cross(mean_angle, np.array([0, 1, 0])) u_dir = normalize(u_dir) v_dir = np.cross(mean_angle, u_dir) u_proj = np.dot(angles_for_pmt, u_dir) u_var = np.var(u_proj, ddof=1) v_proj = np.dot(angles_for_pmt, v_dir) v_var = np.var(v_proj, ddof=1) variance = (u_var + v_var) / 2. # Old method, which calculated variance of projected norma, even though # the mean of the projections wasn't 0 due to the solid angle factor norms = np.repeat(1.0, n_angles) projection_norms = np.dot(angles_for_pmt, mean_angle) orthogonal_complements = np.sqrt( np.maximum(norms**2 - projection_norms**2, 0.)) #variance = np.var(orthogonal_complements, ddof=1) uvvar = u_var - v_var try: #draw_pmt_ind = None draw_pmt_ind = int(draw_pmt_ind) ''' if i == draw_pmt_ind or draw_pmt_ind<0: # Temporary, to visualize histogram of angles, distances #angles = np.arccos(projection_norms) #ang_variance = np.var(angles, ddof=1) #fig1 = plt.figure(figsize=(7.8, 6)) #plt.hist(angles, bins=20) #plt.xlabel('Angular Separation to Mean Angle') #plt.ylabel('Counts per bin') #plt.title('Angles Histogram for PMT ' + str(i)) ##plt.show() # #fig2 = plt.figure(figsize=(7.8, 6)) #plt.hist(angles, bins=20, weights=1./np.sin(angles)) #plt.xlabel('Angular Separation to Mean Angle') #plt.ylabel('Counts per solid angle') #plt.title('Angles Histogram for PMT ' + str(i)) fig3 = plt.figure(figsize=(7.8, 6)) plt.hist(orthogonal_complements, bins=20) plt.xlabel('Normalized Distance to Mean Angle') plt.ylabel('Counts per bin') plt.title('Distances Histogram for PMT ' + str(i)) fig4 = plt.figure(figsize=(7.8, 6)) plt.hist(u_proj, bins=20) plt.xlabel('U Distance to Mean Angle') plt.ylabel('Counts per bin') plt.title('U Distances Histogram for PMT ' + str(i)) fig5 = plt.figure(figsize=(7.8, 6)) plt.hist(v_proj, bins=20) plt.xlabel('V Distance to Mean Angle') plt.ylabel('Counts per bin') plt.title('V Distances Histogram for PMT ' + str(i)) plt.show() #print "Average projected variance: ", variance #print "Variance of projected 2D norms: ", np.var(orthogonal_complements, ddof=1) draw_pmt_ind = raw_input("Enter index of next PMT to draw; will stop drawing if not a valid PMT index.\n") ''' except ValueError: pass except TypeError: pass total_means[i] = mean_angle total_variances[i] = variance total_u_minus_v[i] = np.abs(uvvar) amount_of_hits[i] = n_angles if np.isnan(variance): print "Nan for PMT " + str(i) # nan_ind = np.where(np.isnan(orthogonal_complements)) # print nan_ind # print projection_norms # print orthogonal_complements # print angles_for_pmt # print variance # print mean_angle # temporary, for debugging: n_hits = np.sum(amount_of_hits, axis=0) print "Total hits for calibrated PMTs: " + str(n_hits) print "PMTs w/ < n_events hits: " + str( len(np.where(amount_of_hits < nevents)[0]) * 1.0 / self.npmt_bins) print "PMTs w/ < n_min hits: " + str( len(np.where(amount_of_hits < n_min)[0]) * 1.0 / self.npmt_bins) print "PMTs w/ < 100 hits: " + str( len(np.where(amount_of_hits < 100)[0]) * 1.0 / self.npmt_bins) print "Mean U-V variance (abs): " + str(np.mean(total_u_minus_v)) # Store final calibrated values self.means = -total_means.astype(np.float32).T self.sigmas = np.sqrt(total_variances.astype(np.float32))