def calculate_one_run(inpath, outpath): hough_responses = [] run = ps.EventListReader(inpath) number_muons = 0 for event in run: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 cherenkov_point_cloud = photon_clusters.point_cloud cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask] point_positions = cherenkov_clusters[:, 0:2] muon_props = detection(event, photon_clusters) if muon_props["is_muon"]: cx = muon_props["muon_ring_cx"] cy = muon_props["muon_ring_cy"] r = muon_props["muon_ring_r"] total_amplitude = evaluate_ring(point_positions, cx, cy, r) hough_responses.append(total_amplitude) number_muons += 1 hough_responses = np.multiply(hough_responses, 100) psf_values = calculate_PSF(hough_responses) psf_error = psf_values * 1 / np.sqrt(number_muons) average_psf = float(np.average(psf_values)) outdir = os.path.dirname(outpath) os.makedirs(outdir, exist_ok=True) with open(outpath + ".temp", "wt") as fout: out = { "average_psf": float(np.average(psf_values)), "psf_stdev": float(np.std(psf_values)), "standard_error": np.average(psf_error), "number_muons": number_muons } fout.write(json.dumps(out)) os.rename(outpath + ".temp", outpath) return 0
def run_job(inpath, outpath, method=False): results = [] run = ps.EventListReader(inpath) number_muons = 0 for event in run: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 cherenkov_point_cloud = photon_clusters.point_cloud cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask] point_positions = cherenkov_clusters[:, 0:2] random_state = np.random.get_state() np.random.seed(event.photon_stream.number_photons) if not callable(method): muon_props = extraction.detection(event, photon_clusters) else: muon_props = method(event, photon_clusters) muon_props = extraction.detection(event, photon_clusters) np.random.set_state(random_state) if muon_props["is_muon"]: cx = muon_props["muon_ring_cx"] cy = muon_props["muon_ring_cy"] r = muon_props["muon_ring_r"] total_amplitude = evaluate_ring(point_positions, cx, cy, r) results.append(total_amplitude) number_muons += 1 outdir = os.path.dirname(outpath) os.makedirs(outdir, exist_ok=True) with open(outpath + ".temp", "wt") as fout: out = { "average_fuzz": float(np.average(results)), "std_fuzz": float(np.std(results)), "number_muons": number_muons, } fout.write(json.dumps(out)) os.rename(outpath + ".temp", outpath)
def extract_muons_from_run(self): run = ps.EventListReader(self.simulationFile) event_infos = [] for i, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_features = detection(event, photon_clusters) if muon_features["is_muon"]: event_id = i muon_ring_cx = muon_features['muon_ring_cx'] muon_ring_cy = muon_features['muon_ring_cy'] muon_ring_r = muon_features['muon_ring_r'] mean_arrival_time_muon_cluster = muon_features[ 'mean_arrival_time_muon_cluster'] muon_ring_overlapp_with_field_of_view = muon_features[ 'muon_ring_overlapp_with_field_of_view'] number_of_photons = muon_features['number_of_photons'] event_info = [ event_id, muon_ring_cx, muon_ring_cy, muon_ring_r, mean_arrival_time_muon_cluster, muon_ring_overlapp_with_field_of_view, number_of_photons ] header = list([ "event_id", "muon_ring_cx", "muon_ring_cy", "muon_ring_r", "mean_arrival_time_muon_cluster", "muon_ring_overlapp_with_field_of_view", "number_of_photons" ]) headers = ",".join(header) event_infos.append(event_info) np.savetxt(self.extracted_muons, event_infos, delimiter=",", comments='', header=headers)
def analysis_main(self): run = ps.EventListReader(self.simulationFile) ringModel_event_infos = [] medianR_event_infos = [] knownC_event_infos = [] hough_event_infos = [] for event_id, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) hough_muonFeatures = detection(event, photon_clusters) ringM_muonFeatures = dwsrf(event, photon_clusters) if hough_muonFeatures["is_muon"]: hough_event_info = self.extract_with_hough( event_id, hough_muonFeatures) hough_event_infos.append(hough_event_info) if ringM_muonFeatures["is_muon"]: ringModel_event_info = self.only_ringModel( event_id, ringM_muonFeatures) ringModel_event_infos.append(ringModel_event_info) medianR_event_info = self.medianR_extraction( ringM_muonFeatures, photon_clusters, event_id) medianR_event_infos.append(medianR_event_info) knownC_event_info = self.known_center(event_id, photon_clusters) knownC_event_infos.append(knownC_event_info) data_to_be_saved = [ ringModel_event_infos, medianR_event_infos, knownC_event_infos, hough_event_infos ] methods = ["ringM", "medianR", "knownC", "hough"] for method, data in zip(methods, data_to_be_saved): self.save_to_file(data, method)
def run_job(inpath, outpath, method=False): results = [] run = ps.EventListReader(inpath) number_muons = 0 for event in run: clusters = ps.PhotonStreamCluster(event.photon_stream) random_state = np.random.get_state() np.random.seed(event.photon_stream.number_photons) if not callable(method): muon_props = extraction.detection(event, clusters) else: muon_props = method(event, clusters) np.random.set_state(random_state) if muon_props["is_muon"]: std_photons_on_ring = muon_ring_std_event(clusters, muon_props) results.append(std_photons_on_ring) number_muons += 1 outdir = os.path.dirname(outpath) os.makedirs(outdir, exist_ok=True) with open(outpath + ".temp", "wt") as fout: out = { "average_fuzz": float(np.average(results)), "std_fuzz": float(np.std(results)), "number_muons": number_muons, } fout.write(json.dumps(out)) os.rename(outpath + ".temp", outpath)
def gen_features_norm(data_file, lower, upper, sim_file=None): """ This generates a certain set of features from photon-stream simulation or data files that can be used for further analyses. Inputs: data_file: location of input data file as string lower: lower limit for time slice cleaning upper: upper limit for time slice cleaning sim_file: location of input simulations file as string default: corresponding to name of data file return: pandas data frame with features """ # read in files if is_simulation_file(data_file): reader = ps.SimulationReader( photon_stream_path=data_file, mmcs_corsika_path=sim_file) else: reader = ps.EventListReader(data_file) # initialisation of list of dicts containing generated data events = list() border_pix = get_border_pixel_mask() x, y = get_pixel_coords() # loop for events for event in reader: lol = event.photon_stream.list_of_lists image = phs2image(lol, lower, upper) mask = facttools_cleaning(image, lol, lower, upper) # empty dict for values ev = {} # number of photons in biggest cluster ev['size'] = image[mask].sum() if ev['size'] > 0: border_ph = [(border_pix[i] and mask[i]) for i in range(1440)] ev['leakage'] = image[border_ph].sum()/ev['size'] ev.update(safe_observation_info(event)) ev.update(calc_hillas_features_image(image, mask)) # append values from dict to list of dicts (events) events.append(ev) # save list of dicts in pandas data frame df = pd.DataFrame(events) return df
def process(self): used_paths = split_data(self.raw_paths)[self.split] for raw_path in used_paths: if not self.include_proton and "proton" in raw_path: continue # load the pickled file from the disk if osp.exists(osp.join(self.processed_dir, self.split, f"{raw_path}.pt")): self.processed_filenames.append(f"{raw_path}.pt") else: if self.simulated: mc_truth = raw_path.split(".phs")[0] + ".ch.gz" event_reader = ps.SimulationReader( photon_stream_path=raw_path, mmcs_corsika_path=mc_truth ) else: event_reader = ps.EventListReader(raw_path) for event in event_reader: print(raw_path) # Convert List of List to Point Cloud, then truncation is simply cutting in the z direction point_cloud = np.asarray(event.photon_stream.point_cloud) # Read data from `raw_path`. data = Data( pos=point_cloud ) # Just need x,y,z ignore derived features if self.simulated: if "proton" in raw_path: data.event_type = torch.tensor(0, dtype=torch.int8) elif "gamma" in raw_path: data.event_type = torch.tensor(1, dtype=torch.int8) else: print("No Event Type") continue data.energy = torch.tensor( event.simulation_truth.air_shower.energy, dtype=torch.float ) data.phi = torch.tensor( event.simulation_truth.air_shower.phi, dtype=torch.float ) data.theta = torch.tensor( event.simulation_truth.air_shower.theta, dtype=torch.float ) if self.pre_filter is not None and not self.pre_filter(data): continue if self.pre_transform is not None: data = self.pre_transform(data) torch.save( data, osp.join( self.processed_dir, self.split, "{}.pt".format(raw_path) ), ) self.processed_filenames.append("{}.pt".format(raw_path))
def main(): gammas = list(filter(lambda a: a.endswith('phs.jsonl.gz'), os.listdir(gamma_path))) protons = list(filter(lambda a: a.endswith('phs.jsonl.gz'), os.listdir(proton_path))) data = ps.EventListReader('/net/big-tank/POOL/projects/fact/photon-stream/stream_data/crab/20131101_160.phs.jsonl.gz') print('Reading data...') lol_d = [] events = 0 for event in tqdm(data, total=2000): events += 1 plot_time(lol_d, event) if events > 2000: break print('Reading gamma...') lol_g = [] events = 0 for file in gammas: gamma = ps.EventListReader(gamma_path + file) for event in tqdm(gamma, total=2000): events += 1 plot_time(lol_g, event) if events > 2000: break if events > 2000: break print('Reading proton...') lol_p = [] events = 0 for file in protons: proton = ps.EventListReader(proton_path + file) for event in tqdm(proton, total=2000): events += 1 plot_time(lol_p, event) if events > 2000: break if events > 2000: break safe_plot(lol_d, lol_g, lol_p)
def extract_fuzzParam_from_single_run(self, inpath): response_resultsR = [] response_resultsH = [] fuzz_resultsR = [] fuzz_resultsH = [] number_muonsH = 0 number_muonsR = 0 mu_event_ids = [] reconstructed_muon_eventsH = [] reconstructed_muon_eventsR = [] run = ps.EventListReader(inpath) for event_id, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_propsR = ringM(event, photon_clusters) muon_propsH = hough(event, photon_clusters) if muon_propsH['is_muon']: houghResults = self.extraction(muon_propsH, photon_clusters, event_id) reconstructed_muon_eventsH.append(houghResults[2]) response_resultsH.append(houghResults[0]) fuzz_resultsH.append(houghResults[1]) number_muonsH += 1 if muon_propsR['is_muon']: ringM_results = self.extraction(muon_propsR, photon_clusters, event_id) reconstructed_muon_eventsR.append(ringM_results[2]) response_resultsR.append(ringM_results[0]) fuzz_resultsR.append(ringM_results[1]) number_muonsR += 1 psf = self.get_point_spread_function(inpath) responseR_avg, responseR_stdev = self.calculate_average_and_stdev( response_resultsR) fuzzR_avg, fuzzR_stdev = self.calculate_average_and_stdev( fuzz_resultsR) responseH_avg, responseH_stdev = self.calculate_average_and_stdev( response_resultsH) fuzzH_avg, fuzzH_stdev = self.calculate_average_and_stdev( fuzz_resultsH) runInfo = { "reconstructed_muon_eventsH": reconstructed_muon_eventsH, "responseR_avg": responseR_avg, "responseR_stdev": responseR_stdev, "fuzzR_avg": fuzzR_avg, "fuzzR_stdev": fuzzR_stdev, "number_muonsH": number_muonsH, "reconstructed_muon_eventsR": reconstructed_muon_eventsR, "responseH_avg": responseH_avg, "responseH_stdev": responseH_stdev, "fuzzH_avg": fuzzH_avg, "fuzzH_stdev": fuzzH_stdev, "number_muonsR": number_muonsR, "point_spread_function": psf, "inpath": inpath } return runInfo
def cluster_single_run(dir_name, clustering=ps.PhotonStreamCluster): nsb_run_photons = [] pure_run_photons = [] all_run_photons = [] pure_run_path = os.path.join(dir_name, "pure", "psf_0.sim.phs") nsb_run_path = os.path.join(dir_name, "NSB", "psf_0.sim.phs") nsb_run = ps.EventListReader(nsb_run_path) pure_run = ps.EventListReader(pure_run_path) for event in nsb_run: photon_clusters = clustering(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 nsb_cherenkov_photon_stream = photon_clusters.point_cloud nsb_cherenkov_ps = nsb_cherenkov_photon_stream[cherenkov_cluster_mask] nsb_run_photons.append(nsb_cherenkov_ps[:, 0:3]) all_photons = event.photon_stream.point_cloud all_run_photons.append(all_photons) for muon in pure_run: pure_photon_stream = muon.photon_stream.point_cloud pure_run_photons.append(pure_photon_stream) return (all_run_photons, pure_run_photons, nsb_run_photons)
def do_clustering(self, pure_cherenkov_events_path, events_with_nsb_path): pure_cherenkov_run = ps.EventListReader(pure_cherenkov_events_path) nsb_run = ps.EventListReader(events_with_nsb_path) nsb_run_found_photons = [] pure_cherenkov_run_photons = [] cut_cherenkov = [] all_photons_run = [] for event in nsb_run: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 nsb_cherenkov_photon_stream = photon_clusters.point_cloud nsb_cherenkov_ps = nsb_cherenkov_photon_stream[ cherenkov_cluster_mask] cherenkov_photons = self.cut_hist(nsb_cherenkov_ps) cut_cherenkov.append(cherenkov_photons) nsb_run_found_photons.append(nsb_cherenkov_ps[:, 0:3]) all_photons = event.photon_stream.point_cloud all_photons_run.append(all_photons) for muon in pure_cherenkov_run: pure_photon_stream = muon.photon_stream.point_cloud pure_cherenkov_run_photons.append(pure_photon_stream) return (all_photons_run, pure_cherenkov_run_photons, nsb_run_found_photons, cut_cherenkov)
def gen_features(data_file, eps=0.1, sim_file=None): """ This generates a certain set of features from photon-stream simulation or data files that can be used for further analyses. Inputs: ----------------------------------------- data_file: location of input data file as string sim_file: location of input simulations file as string default: corresponding to name of data file Returns: ----------------------------------------- pandas data frame with features """ # if eps is None: # eps = 0.1 # read in files if is_simulation_file(data_file): reader = ps.SimulationReader( photon_stream_path=data_file, mmcs_corsika_path=sim_file) else: reader = ps.EventListReader(data_file) # initialisation of list of dicts containing generated data events = list() # loop for events for event in reader: # clustering of events clustering = ps.photon_cluster.PhotonStreamCluster(event.photon_stream, eps=eps) if clustering.number >= 1: # empty dict for values ev = {} # safe hillas features ev.update(calc_hillas_features_phs(event.photon_stream, clustering)) # safe observation info ev.update(safe_observation_info(event)) # append values from dict to list of dicts (events) events.append(ev) # save list of dicts in pandas data frame df = pd.DataFrame(events) return df
def run_detection(self, inpath): run = ps.EventListReader(inpath) path = os.path.normpath(inpath) split_path = path.split(os.sep) oa_name = split_path[-2] oa = re.split('_', oa_name)[2] preferences = self.read_preferencesFile() number_of_muons = preferences['--number_of_muons'] found_muons = 0 for event in run: clusters = ps.PhotonStreamCluster(event.photon_stream) muon_props = detection(event, clusters) if muon_props["is_muon"]: found_muons += 1 return oa, found_muons, number_of_muons
def convert_file(path): # print('Analyzing {}'.format(photon_stream_file)) try: reader = ps.SimulationReader( photon_stream_path=path, ) except Exception as e: print(e) reader = ps.EventListReader(path) try: rows = [image_from_event(event) for event in reader] return Table(rows).as_array() except ValueError as e: print('Failed to read data from file: {}'.format(path)) print('PhotonStreamError: {}'.format(e))
def test_muon_detection(): np.random.seed(seed=1) muon_truth_path = pkg_resources.resource_filename( 'muons', os.path.join('tests', 'resources', 'muon_sample_20140101_104.csv')) muon_truth = np.genfromtxt(muon_truth_path) muon_sample_path = pkg_resources.resource_filename( 'muons', os.path.join('tests', 'resources', '20140101_104_muon_sample.phs.jsonl.gz')) run = ps.EventListReader(muon_sample_path) true_positives = 0 true_negatives = 0 false_positives = 0 false_negatives = 0 for event in run: clusters = ps.PhotonStreamCluster(event.photon_stream) ret = muons.detection(event, clusters) if ret['is_muon']: if event.observation_info.event in muon_truth: true_positives += 1 else: false_positives += 1 else: if event.observation_info.event in muon_truth: false_negatives += 1 else: true_negatives += 1 precision = true_positives / (true_positives + false_positives) sensitivity = true_positives / (true_positives + false_negatives) print('precision', precision) print('sensitivity', sensitivity) assert precision > 0.995 assert sensitivity > 0.76
def test_extraction(): Analysis = pfa.PSF_FuzzAnalysis( preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir) simulationFile = os.path.join( fileDir, "resources", "100simulations_psf0.0.sim.phs") run = ps.EventListReader(simulationFile) event_id = 3 for i, event in enumerate(run): if i == event_id: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_props = { "muon_ring_cx": 0, "muon_ring_cy": 0, "muon_ring_r": 1, "mean_arrival_time_muon_cluster": 2, "muon_ring_overlapp_with_field_of_view": 3, "number_of_photons": 3 } returns = Analysis.extraction( muon_props, photon_clusters, event_id) assert len(returns) == 3
def test_get_fuzziness_parameters(): Analysis = pfa.PSF_FuzzAnalysis( preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir) simulationFile = os.path.join( fileDir, "resources", "100simulations_psf0.0.sim.phs") run = ps.EventListReader(simulationFile) for i, event in enumerate(run): if i == 47: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_props = { "muon_ring_cx": 0, "muon_ring_cy": 0, "muon_ring_r": 1, "mean_arrival_time_muon_cluster": 2, "muon_ring_overlapp_with_field_of_view": 3, "number_of_photons": 3 } normed_response, fuzziness_stdParam = Analysis.get_fuzziness_parameters( photon_clusters, muon_props) assert ( isinstance(normed_response, Number) and isinstance(fuzziness_stdParam, Number) )
def count_events(self): if self.num_events < 0: count = 0 for index, file in enumerate(self.paths): try: print("Trying...") crab_reader = ps.EventListReader(file) for event in crab_reader: df_event = self.dl2_file.loc[ (self.dl2_file["event_num"] == event.observation_info.event) & (self.dl2_file["night"] == event.observation_info.night) & (self.dl2_file["run_id"] == event.observation_info.run) ] if not df_event.empty: count += 1 print(count) except Exception as e: print(str(e)) print(count) print("\n") self.num_events = count return count return self.num_events
def from_observation(phs_path): event_list = ps.EventListReader(phs_path) features = [] for event in event_list: if event.observation_info.trigger_type == PHYSICS_TRIGGER: cluster = ps.PhotonStreamCluster(event.photon_stream) cluster = reject.early_or_late_clusters(cluster) f = raw_features(photon_stream=event.photon_stream, cluster=cluster) f['type'] = ps.io.binary.OBSERVATION_EVENT_TYPE_KEY f['az'] = np.deg2rad(event.az) f['zd'] = np.deg2rad(event.zd) f['night'] = event.observation_info.night f['run'] = event.observation_info.run f['event'] = event.observation_info.event f['time'] = event.observation_info._time_unix_s + event.observation_info._time_unix_us / 1e6 features.append(f) triggered = pd.DataFrame(features) triggered = tools.reduce_DataFrame_to_32_bit(triggered) return triggered
def main(method, path, file, feat, number): border_pix = get_border_pixel_mask() print("Reading in facttools dl1 file...") t = Table.read( '/net/big-tank/POOL/projects/fact/photon-stream/facttools/crab/{}_dl1.fits' .format(file)) # print("Reading in facttools dl2 file...") # dl2 = read_data('/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/crab.hdf5', key='events') print("Reading in PhotonStream data file...") reader = ps.EventListReader( '/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz' .format(path, file)) print("Reading in PhotonStream dl2 data...") ph_dl2 = read_data('/home/ksedlaczek/OUT_0.10/crab_data_precuts.hdf5', key='events') print("Done...") event = next(reader) d_delta = [] disp_prediction = [] for i in tqdm(range(number)): if is_simulation_event(event): event_num_phs = event.simulation_truth.event reuse_phs = event.simulation_truth.reuse run_id_phs = event.simulation_truth.run else: event_num_phs = event.observation_info.event reuse_phs = 42 run_id_phs = event.observation_info.run if path != 'crab': run_id = file event_num = t[i]['MCorsikaEvtHeader.fEvtNumber'] reuse = t[i]['MCorsikaEvtHeader.fNumReuse'] else: run_id = file event_num = t[i]['EventNum'] reuse = 42 assert run_id != run_id_phs while (event_num_phs != event_num or reuse != reuse_phs): event = next(reader) if is_simulation_event(event): event_num_phs = event.simulation_truth.event reuse_phs = event.simulation_truth.reuse run_id_phs = event.simulation_truth.run else: event_num_phs = event.observation_info.event reuse_phs = 42 run_id_phs = event.observation_info.run lol = event.photon_stream.list_of_lists # cut away unwanted time slices # lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol] image = phs2image(lol) disp_prediction.append( ph_dl2.query('night == 20131104 & run == 162 & event == {}'.format( event_num))['disp_prediction'].values) cleaned_pix = t[i]['shower'] d_delta.append(calc_delta_delta(event, cleaned_pix)) mask = np.array([ True if disp_prediction[i] < 0 else False for i in range(len(disp_prediction)) ]) plt.figure() plt.hist(np.array(d_delta)[mask], bins=np.linspace(-np.pi / 2, 3 / 2 * np.pi, 101), alpha=0.4, label='disp < 0') plt.hist(np.array(d_delta)[~mask], bins=np.linspace(-np.pi / 2, 3 / 2 * np.pi, 101), alpha=0.4, label='disp >= 0') # plt.hist(np.array(d_delta)[mask], bins=100, alpha=0.4, label='disp < 0') # plt.hist(np.array(d_delta)[~mask], bins=100, alpha=0.4, label='disp >= 0') plt.title( r'$\mathrm{\Delta}\delta$ between phs on facttools cleaning to $\delta_{\mathrm{true}}$ per image' ) plt.legend(loc='best') plt.xlabel(r'$\mathrm{\Delta}\delta$ / rad') plt.savefig('delta_true_diff_hist_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf()
def run_job(self, job): results = [] inpath = job["inpath"] output_path_responseH = job["output_path_responseH"] output_path_stdevR = job["output_path_stdevR"] output_path_responseR = job["output_path_responseR"] output_path_stdevH = job["output_path_stdevH"] muonCountH = 0 muonCountR = 0 run = ps.EventListReader(inpath) muon_ring_featuresR = [] muon_ring_featuresH = [] fuzziness_stdParamRs = [] fuzziness_stdParamHs = [] normed_responseRs = [] normed_responseHs = [] fuzz_paramsH = [] for event_id, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_propsR = ringM_detection(event, photon_clusters) muon_propsH = detection(event, photon_clusters) if muon_propsH["is_muon"]: muonCountH += 1 normed_responseH, fuzziness_stdParamH = ( self.get_fuzziness_parameters(photon_clusters, muon_propsH)) muon_ring_featureH = [ muon_propsH["muon_ring_cx"], muon_propsH["muon_ring_cy"], muon_propsH["muon_ring_r"] ] muon_ring_featuresH.append(muon_ring_featureH) fuzziness_stdParamHs.append(fuzziness_stdParamH) normed_responseHs.append(normed_responseH) if muon_propsR["is_muon"]: muonCountR += 1 normed_responseR, fuzziness_stdParamR = ( self.get_fuzziness_parameters(photon_clusters, muon_propsR)) muon_ring_featureR = [ muon_propsR["muon_ring_cx"], muon_propsR["muon_ring_cy"], muon_propsR["muon_ring_r"] ] fuzziness_stdParamRs.append(fuzziness_stdParamR) normed_responseRs.append(normed_responseR) muon_ring_featuresR.append(muon_ring_featureR) fact_path = fact.path.parse(inpath) night = fact_path["night"] run = fact_path["run"] filename = str(night) + "_" + str(run) + ".csv" output_dirR = os.path.dirname(output_path_stdevR) output_dirH = os.path.dirname(output_path_stdevH) if not os.path.isdir(output_dirR): os.makedirs(output_dirR, exist_ok=True) if not os.path.isdir(output_dirH): os.makedirs(output_dirH, exist_ok=True) self.save_to_file("ringM", output_dirR, filename, muon_ring_featuresR) self.save_to_file("hough", output_dirH, filename, muon_ring_featuresH) self.save_fuzz_param(output_path_stdevR, fuzziness_stdParamRs, muonCountR) self.save_fuzz_param(output_path_responseR, normed_responseRs, muonCountR) self.save_fuzz_param(output_path_stdevH, fuzziness_stdParamHs, muonCountH) self.save_fuzz_param(output_path_responseH, normed_responseHs, muonCountH) return 0
def main(method, path, file, feat, number): border_pix = get_border_pixel_mask() print("Reading in facttools dl1 file...") t = Table.read('/net/big-tank/POOL/projects/fact/photon-stream/facttools/crab/{}_dl1.fits'.format(file)) print("Reading in facttools dl2 file...") dl2 = read_data('/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/crab.hdf5', key='events') print("Reading in PhotonStream data file...") reader = ps.EventListReader('/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'.format(path, file)) print("Done...") event = next(reader) fig, axs = plt.subplots(3, 1, figsize=(4, 10), constrained_layout=True) plots = [camera(np.zeros(1440), cmap='inferno', ax=ax) for ax in axs] cbars = [fig.colorbar(plot, ax=ax) for plot, ax in zip(plots, axs)] plots[1].set_cmap('RdBu_r') with PdfPages('pe_difference_{}_{}.pdf'.format(feat, file)) as pdf: for i in tqdm(range(number)): if is_simulation_event(event): event_num_phs = event.simulation_truth.event reuse_phs = event.simulation_truth.reuse run_id_phs = event.simulation_truth.run else: event_num_phs = event.observation_info.event reuse_phs = 42 run_id_phs = event.observation_info.run if path != 'crab': run_id = file event_num = t[i]['MCorsikaEvtHeader.fEvtNumber'] reuse = t[i]['MCorsikaEvtHeader.fNumReuse'] else: run_id = file event_num = t[i]['EventNum'] reuse = 42 assert run_id != run_id_phs while (event_num_phs != event_num or reuse != reuse_phs): event = next(reader) if is_simulation_event(event): event_num_phs = event.simulation_truth.event reuse_phs = event.simulation_truth.reuse run_id_phs = event.simulation_truth.run else: event_num_phs = event.observation_info.event reuse_phs = 42 run_id_phs = event.observation_info.run lol = event.photon_stream.list_of_lists # cut away unwanted time slices # lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol] image = phs2image(lol) if method == 'DBSCAN': clustering = ps.photon_cluster.PhotonStreamCluster(event.photon_stream) biggest_cluster = np.argmax(np.bincount(clustering.labels[clustering.labels != -1])) mask = clustering.labels == biggest_cluster cleaned_pix_phs = np.zeros(len(image), dtype=bool) k = 0 cleaned_img = np.zeros(len(image)) for h in range(len(lol)): for j in range(len(lol[h])): k += 1 if mask[k-1]: cleaned_pix_phs[h] = True else: cleaned_pix_phs = facttools_cleaning(image, lol, picture_thresh, boundary_thresh) if sum(cleaned_pix_phs) < 1: break cleaned_pix = t[i]['shower'] t[i]['photoncharge'][t[i]['photoncharge'] < 0] = 0.0 pe_difference = image - t[i]['photoncharge'] max_abs = np.max(np.abs(pe_difference)) plots[0].set_array(image) plots[1].set_array(pe_difference) plots[2].set_array(t[i]['photoncharge']) plots[0].autoscale() plots[1].set_clim(-max_abs, max_abs) plots[2].autoscale() # mark_pixel(t[i]['shower'], color=(128/255, 186/255, 38/255), linewidth=2.5) for ax in axs: ax.axis('off') for cbar, plot in zip(cbars, plots): cbar.update_bruteforce(plot) # embed() if is_simulation_event(event): fig.suptitle('run {} event {} reuse {} mean {:.2f}'.format(run_id, event_num, reuse, np.mean(pe_difference))) else: # fig.suptitle('{} event {} mean {:.2f}'.format(file, event_num, np.mean(pe_difference))) fig.suptitle('{} event {}'.format(file[:8] + ' ' + file[9:], event_num)) pdf.savefig(fig)
def main(method, path, file, feat, number): border_pix = get_border_pixel_mask() print("Reading in facttools dl1 file...") t = Table.read('/net/big-tank/POOL/projects/fact/photon-stream/facttools/{}/{}_dl1.fits'.format(path, file)) print("Reading in facttools dl2 file...") dl2 = read_data('/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/{}.hdf5'.format(path), key='events') print("Reading in PhotonStream data file...") reader = ps.EventListReader('/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'.format(path, file)) print("Done...") event = next(reader) all_pe_diff_mean = [] all_pe_diff = [] delta_delta = [] delta_delta_diff = [] delta_delta_diff_perc = [] d_delta = [] for i in tqdm(range(number)): if is_simulation_event(event): event_num_phs = event.simulation_truth.event reuse_phs = event.simulation_truth.reuse run_id_phs = event.simulation_truth.run else: event_num_phs = event.observation_info.event reuse_phs = 42 run_id_phs = event.observation_info.run if path != 'crab': run_id = file event_num = t[i]['MCorsikaEvtHeader.fEvtNumber'] reuse = t[i]['MCorsikaEvtHeader.fNumReuse'] else: run_id = file event_num = t[i]['EventNum'] reuse = 42 assert run_id != run_id_phs while (event_num_phs != event_num or reuse != reuse_phs): event = next(reader) if is_simulation_event(event): event_num_phs = event.simulation_truth.event reuse_phs = event.simulation_truth.reuse run_id_phs = event.simulation_truth.run else: event_num_phs = event.observation_info.event reuse_phs = 42 run_id_phs = event.observation_info.run lol = event.photon_stream.list_of_lists # cut away unwanted time slices # lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol] image = phs2image(lol) if method == 'DBSCAN': clustering = ps.photon_cluster.PhotonStreamCluster(event.photon_stream) if clustering.number < 1: continue biggest_cluster = np.argmax(np.bincount(clustering.labels[clustering.labels != -1])) mask = clustering.labels == biggest_cluster cleaned_pix_phs = np.zeros(len(image), dtype=bool) k = 0 cleaned_img = np.zeros(len(image)) for h in range(len(lol)): for j in range(len(lol[h])): k += 1 if mask[k-1]: cleaned_pix_phs[h] = True else: cleaned_pix_phs = facttools_cleaning(image, lol, picture_thresh, boundary_thresh) if sum(cleaned_pix_phs) < 1: continue cleaned_pix = t[i]['shower'] t[i]['photoncharge'][t[i]['photoncharge'] < 0] = 0.0 pe_difference = image - t[i]['photoncharge'] if is_simulation_event(event): delta = np.rad2deg(dl2.query('night == 20131104 & run_id == 162 & event_num == {}'.format(t[i]['EventNum']))['delta'].values[0]) else: delta = np.rad2deg(dl2.query('night == 20131104 & run_id == 162 & event_num == {}'.format(t[i]['EventNum']))['delta'].values[0]) delta_diff_same = calc_delta(image, cleaned_pix) - delta delta_diff_whole = calc_delta(image, cleaned_pix_phs) - delta delta_diff_whole_perc = calc_delta_perc(image, cleaned_pix_phs) - delta for val in pe_difference: all_pe_diff.append(val) all_pe_diff_mean.append(np.mean(pe_difference)) delta_delta.append(delta_diff_same) delta_delta_diff.append(delta_diff_whole) delta_delta_diff_perc.append(delta_diff_whole_perc) d_delta.append(calc_delta_delta(event, cleaned_pix)) plt.figure() plt.hist(all_pe_diff_mean, bins=100, histtype='step') plt.title('Means of pe differences per image') plt.tight_layout() plt.savefig('means_hist_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(all_pe_diff, bins=100, histtype='step', density=True) plt.title('PE differences per pixel') plt.tight_layout() plt.savefig('diffs_hist_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(all_pe_diff, bins=100, histtype='step') plt.title('PE differences per pixel') plt.semilogy() plt.ylabel('events') plt.xlabel(r'$\symup{\Delta}$PE') plt.tight_layout() plt.savefig('diffs_hist_{}_{}_{}_logy.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(delta_delta, bins=70, histtype='step', density=True) plt.title('$\mathrm{\Delta}\delta$ between phs and facttools') plt.tight_layout() plt.savefig('delta_hist_same_pixels_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(delta_delta_diff, bins=70, histtype='step', density=True) plt.title(r'$\mathup{\Delta}\delta$ between phs and facttools') plt.xlabel(r'$\mathrm{\Delta}\delta / \textdegree$') plt.tight_layout() plt.savefig('delta_diff_hist_different_cleanings_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(delta_delta_diff_perc, bins=70, histtype='step', density=True) plt.title(r'$\mathup{\Delta}\delta$ between phs and facttools') plt.xlabel(r'$\mathrm{\Delta}\delta / \textdegree$') plt.tight_layout() plt.savefig('delta_diff_hist_perc_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(delta_delta_diff, bins=70, histtype='step', density=True, label='DBSCAN') plt.hist(delta_delta_diff_perc, bins=70, histtype='step', density=True, label='DBSCAN + > 1%') plt.title(r'$\mathup{\Delta}\delta$ between phs and facttools') plt.xlabel(r'$\mathrm{\Delta}\delta / \textdegree$') plt.legend() plt.tight_layout() plt.savefig('delta_diff_hist_perc_and_normal_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf() plt.figure() plt.hist(d_delta, bins=100, histtype='step', density=True) plt.title(r'$\mathrm{\Delta}\delta$ between phs on facttools cleaning to $\delta_{\mathrm{true}}$ per image') plt.tight_layout() plt.savefig('delta_true_diff_hist_{}_{}_{}.pdf'.format(method, feat, file)) plt.clf()
def event_processor( self, directory, clean_images=False, only_core=True, clump_size=20 ): for index, file in enumerate(self.paths): file_name = file.split("/")[-1].split(".phs")[0] try: sim_reader = ps.EventListReader(file) counter = 0 for event in sim_reader: df_event = self.dl2_file.loc[ (self.dl2_file["event_num"] == event.observation_info.event) & (self.dl2_file["night"] == event.observation_info.night) & (self.dl2_file["run_id"] == event.observation_info.run) ] if not df_event.empty: counter += 1 if os.path.isfile( os.path.join( directory, "clump" + str(clump_size), str(file_name) + "_" + str(counter), ) ) and os.path.isfile( os.path.join( directory, "core" + str(clump_size), str(file_name) + "_" + str(counter), ) ): print("True: " + str(file_name) + "_" + str(counter)) continue if clean_images: # Do it for no clumps, all clump, and only core into different subfolders all_photons, clump_photons, core_photons = self.clean_image( event, only_core=only_core, min_samples=clump_size ) if core_photons is None: print("No Clumps, skip") continue else: for key, photon_set in { "no_clean": all_photons, "clump": clump_photons, "core": core_photons, }.items(): event.photon_stream.raw = photon_set # In the event chosen from the file # Each event is the same as each line below source_pos_x = df_event["source_position_x"].values[ 0 ] source_pos_y = df_event["source_position_y"].values[ 0 ] timestamp = ( df_event["timestamp"] .values[0] .astype(datetime.datetime) ) event_photons = event.photon_stream.list_of_lists zd_deg = event.zd az_deg = event.az cog_x = df_event["cog_x"].values[0] cog_y = df_event["cog_y"].values[0] sky_source_az = df_event[ "source_position_az" ].values[0] sky_source_zd = df_event[ "source_position_zd" ].values[0] zd_deg1 = df_event["pointing_position_zd"].values[0] az_deg1 = df_event["pointing_position_az"].values[0] event_num = event.observation_info.event night = event.observation_info.night run = event.observation_info.run data_dict = [ [ event_photons, timestamp, zd_deg, az_deg, cog_x, cog_y, sky_source_az, sky_source_zd, zd_deg1, az_deg1, source_pos_x, source_pos_y, event_num, night, run, ], { "Image": 0, "Timestamp": 1, "Zd_Deg": 2, "Az_Deg": 3, "COG_X": 4, "COG_Y": 5, "Source_Position_Az": 6, "Source_Position_Zd": 7, "Pointing_Position_Zd": 8, "Pointing_Position_Az": 9, "Source_Position_X": 10, "Source_Position_Y": 11, "Event_Number": 12, "Night": 13, "Run": 14, }, ] if key != "no_clean": with open( os.path.join( directory, key + str(clump_size), str(file_name) + "_" + str(counter), ), "wb", ) as event_file: pickle.dump(data_dict, event_file) else: if not os.path.isfile( os.path.join( directory, key, str(file_name) + "_" + str(counter), ) ): with open( os.path.join( directory, key, str(file_name) + "_" + str(counter), ), "wb", ) as event_file: pickle.dump(data_dict, event_file) else: # In the event chosen from the file # Each event is the same as each line below # Each event is the same as each line below source_pos_x = df_event["source_position_x"].values[0] source_pos_y = df_event["source_position_y"].values[0] timestamp = ( df_event["timestamp"] .values[0] .astype(datetime.datetime) ) event_photons = event.photon_stream.list_of_lists zd_deg = event.zd az_deg = event.az cog_x = df_event["cog_x"].values[0] cog_y = df_event["cog_y"].values[0] sky_source_az = df_event["source_position_az"].values[0] sky_source_zd = df_event["source_position_zd"].values[0] zd_deg1 = df_event["pointing_position_zd"].values[0] az_deg1 = df_event["pointing_position_az"].values[0] event_num = event.observation_info.event night = event.observation_info.night run = event.observation_info.run data_dict = [ [ event_photons, timestamp, zd_deg, az_deg, cog_x, cog_y, sky_source_az, sky_source_zd, zd_deg1, az_deg1, source_pos_x, source_pos_y, event_num, night, run, ], { "Image": 0, "Timestamp": 1, "Zd_Deg": 2, "Az_Deg": 3, "COG_X": 4, "COG_Y": 5, "Source_Position_Az": 6, "Source_Position_Zd": 7, "Pointing_Position_Zd": 8, "Pointing_Position_Az": 9, "Source_Position_X": 10, "Source_Position_Y": 11, "Event_Number": 12, "Night": 13, "Run": 14, }, ] with open( os.path.join( directory, str(file_name) + "_" + str(counter) ), "wb", ) as event_file: pickle.dump(data_dict, event_file) except Exception as e: print(str(e)) pass
def single_processor( self, normalize=False, collapse_time=False, final_slices=5, as_channels=False, clean_images=False, ): while True: print("New Crab") for index, file in enumerate(self.paths): try: sim_reader = ps.EventListReader(file) for event in sim_reader: data = [] df_event = self.dl2_file.loc[ (self.dl2_file["event_num"] == event.observation_info.event) & (self.dl2_file["night"] == event.observation_info.night) & (self.dl2_file["run_id"] == event.observation_info.run) ] if not df_event.empty: if clean_images: event = self.clean_image(event) # In the event chosen from the file # Each event is the same as each line below source_pos_x = df_event["source_position_x"].values[0] source_pos_y = df_event["source_position_y"].values[0] energy = ( df_event["timestamp"] .values[0] .astype(datetime.datetime) ) event_photons = event.photon_stream.list_of_lists zd_deg = event.zd az_deg = event.az cog_x = df_event["cog_x"].values[0] cog_y = df_event["cog_y"].values[0] sky_source_az = df_event["source_position_az"].values[0] sky_source_zd = df_event["source_position_zd"].values[0] zd_deg1 = df_event["pointing_position_zd"].values[0] az_deg1 = df_event["pointing_position_az"].values[0] event_num = event.observation_info.event night = event.observation_info.night run = event.observation_info.run input_matrix = np.zeros( [self.shape[1], self.shape[2], self.shape[3]] ) chid_to_pixel = self.rebinning[0] pixel_index_to_grid = self.rebinning[1] for index in range(1440): for element in chid_to_pixel[index]: coords = pixel_index_to_grid[element[0]] for value in event_photons[index]: if self.end >= value >= self.start: input_matrix[coords[0]][coords[1]][ value - self.start ] += (element[1] * 1) data.append( [ np.fliplr(np.rot90(input_matrix, 3)), energy, zd_deg, az_deg, source_pos_x, source_pos_y, sky_source_zd, sky_source_az, zd_deg1, az_deg1, event_num, night, run, cog_x, cog_y, ] ) # need to do the format thing here, and add auxiliary structure data_format = { "Image": 0, "Timestamp": 1, "Zd_Deg": 2, "Az_Deg": 3, "Source_X": 4, "Source_Y": 5, "Theta": 6, } data = self.format(data) if normalize: data = list(data) data[0] = self.normalize_image(data[0]) data = tuple(data) if collapse_time: data = list(data) data[0] = self.collapse_image_time( data[0], final_slices, as_channels ) data = tuple(data) yield data, data_format except Exception as e: print(str(e))
def batch_processor(self, clean_images=False): self.init() for index, file in enumerate(self.paths): print(file) try: sim_reader = ps.EventListReader(file) data = [] for event in sim_reader: df_event = self.dl2_file.loc[ (self.dl2_file["event_num"] == event.observation_info.event) & (self.dl2_file["night"] == event.observation_info.night) & (self.dl2_file["run_id"] == event.observation_info.run) ] if not df_event.empty: if clean_images: event = self.clean_image(event) # In the event chosen from the file # Each event is the same as each line below source_pos_x = df_event["source_position_x"].values[0] source_pos_y = df_event["source_position_y"].values[0] energy = ( df_event["timestamp"].values[0].astype(datetime.datetime) ) event_photons = event.photon_stream.list_of_lists zd_deg = event.zd az_deg = event.az cog_x = df_event["cog_x"].values[0] cog_y = df_event["cog_y"].values[0] sky_source_az = df_event["source_position_az"].values[0] sky_source_zd = df_event["source_position_zd"].values[0] zd_deg1 = df_event["pointing_position_zd"].values[0] az_deg1 = df_event["pointing_position_az"].values[0] event_num = event.observation_info.event night = event.observation_info.night run = event.observation_info.run input_matrix = np.zeros( [self.shape[1], self.shape[2], self.shape[3]] ) chid_to_pixel = self.rebinning[0] pixel_index_to_grid = self.rebinning[1] for index in range(1440): for element in chid_to_pixel[index]: coords = pixel_index_to_grid[element[0]] for value in event_photons[index]: if self.end >= value >= self.start: input_matrix[coords[0]][coords[1]][ value - self.start ] += (element[1] * 1) data.append( [ np.fliplr(np.rot90(input_matrix, 3)), energy, zd_deg, az_deg, source_pos_x, source_pos_y, sky_source_zd, sky_source_az, zd_deg1, az_deg1, event_num, night, run, cog_x, cog_y, ] ) yield data except Exception as e: print(str(e))
def is_simulation_file(input_file): reader = ps.EventListReader(input_file) event = next(reader) return hasattr(event, 'simulation_truth')
def extract_muons_from_run( input_run_path, output_run_path, output_run_header_path ): """ Detects and extracts muon candidate events from a run. The muon candidate events are exported into a new output run. In addidion a header for the muon candidates is exported. Parameter --------- input_run_path Path to the input run. output_run_path Path to the output run of muon candidates. output_run_header_path Path to the binary output run header. Binary Output Format Run Header ------------------------------- for each muon candidate: 1) uint32 Night 2) uint32 Run ID 3) uint32 Event ID 4) uint32 unix time seconds [s] 5) uint32 unix time micro seconds modulo full seconds [us] 6) float32 Pointing zenith distance [deg] 7) float32 Pointing azimuth [deg] 8) float32 muon ring center x [deg] 9) float32 muon ring center y [deg] 10) float32 muon ring radius [deg] 11) float32 mean arrival time muon cluster [s] 12) float32 muon ring overlapp with field of view (0.0 to 1.0) [1] 13) float32 number of photons muon cluster [1] """ run = ps.EventListReader(input_run_path) with gzip.open(output_run_path, 'wt') as f_muon_run, \ open(output_run_header_path, 'wb') as f_muon_run_header: for event in run: if ( event.observation_info.trigger_type == FACT_PHYSICS_SELF_TRIGGER ): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_features = detection(event, photon_clusters) if muon_features['is_muon']: # EXPORT EVENT in JSON event_dict = ps.io.jsonl.event_to_dict(event) json.dump(event_dict, f_muon_run) f_muon_run.write('\n') # EXPORT EVENT header head1 = np.zeros(5, dtype=np.uint32) head1[0] = event.observation_info.night head1[1] = event.observation_info.run head1[2] = event.observation_info.event head1[3] = event.observation_info._time_unix_s head1[4] = event.observation_info._time_unix_us head2 = np.zeros(8, dtype=np.float32) head2[0] = event.zd head2[1] = event.az head2[2] = muon_features['muon_ring_cx']*rad2deg head2[3] = muon_features['muon_ring_cy']*rad2deg head2[4] = muon_features['muon_ring_r']*rad2deg head2[5] = muon_features['mean_arrival_time_muon_cluster'] head2[6] = muon_features[ 'muon_ring_overlapp_with_field_of_view' ] head2[7] = muon_features['number_of_photons'] f_muon_run_header.write(head1.tobytes()) f_muon_run_header.write(head2.tobytes())
def main(method, path, file, feat, number): border_pix = get_border_pixel_mask() if method == "thresholds": reader = ps.EventListReader( '/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz' .format(path, file)) with PdfPages('cleaning_thresh_{}_{}.pdf'.format(feat, file)) as pdf: for i in tqdm(range(number)): fig = plt.figure() ax = fig.add_axes([0.05, 0.05, 0.9, 0.9]) #ax.set_axis_off() event = next(reader) lol = event.photon_stream.list_of_lists lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol] image = phs2image(lol) #, lower=30, upper=70) cleaned_pix = facttools_cleaning(image, lol, 35, 75, picture_thresh, boundary_thresh) with warnings.catch_warnings(): warnings.simplefilter("ignore") arrival_times = np.array([np.nanmedian(l) for l in lol]) # cleaned_pix = cleaning(image, lol, picture_thresh, boundary_thresh) if len(cleaned_pix[cleaned_pix != 0]) > 1: # border_ph = [(border_pix[i] and cleaned_pix[i]) for i in range(1440)] # leakage = image[border_ph].sum()/image[cleaned_pix].sum() df = calc_hillas_features_image(image, cleaned_pix) # ell = Ellipse( # [df['cog_x'], df['cog_y']], # df['length']*2, # df['width']*2, # angle=np.rad2deg(df['delta']), # fill=False, linewidth=2, color='b' # ) # ax.add_patch(ell) ell = Ellipse([df['cog_x'], df['cog_y']], df['length'] * 4, df['width'] * 4, angle=np.rad2deg(df['delta']), fill=False, linewidth=1.5, color='b') # ax.add_patch(ell) if is_simulation_event(event): fig.suptitle('run {} event {} reuse {}'.format( event.simulation_truth.run, event.simulation_truth.event, event.simulation_truth.reuse)) else: fig.suptitle('{} event {} delta {}'.format( file, event.observation_info.event, df['delta'])) if feat == 'arrival_times': with warnings.catch_warnings(): warnings.simplefilter("ignore") x = arrival_times - np.nanmean(arrival_times) x[np.isnan(x)] = 0 c = camera(x, cmap='Spectral', ax=ax) mark_pixel(cleaned_pix, color='k', linewidth=2.5) else: c = camera(image, cmap='viridis', ax=ax) mark_pixel(cleaned_pix, color=(128 / 255, 186 / 255, 38 / 255), linewidth=2.5) ax.axis('off') fig.colorbar(c) pdf.savefig(fig) ax.cla() plt.close(fig) if method == "DBSCAN": reader = ps.EventListReader( '/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz' .format(path, file)) with PdfPages('cleaning_DBSCAN_biggest_{}_{}.pdf'.format(feat, file)) as pdf: for i in tqdm(range(number)): fig = plt.figure() ax = fig.add_axes([0.05, 0.05, 0.9, 0.9]) event = next(reader) # clustering of events clustering = ps.photon_cluster.PhotonStreamCluster( event.photon_stream) if clustering.number > 0: lol = event.photon_stream.list_of_lists image = phs2image(lol) with warnings.catch_warnings(): warnings.simplefilter("ignore") arrival_times = np.array( [np.nanmedian(l) for l in lol]) # biggest cluster: biggest_cluster = np.argmax( np.bincount( clustering.labels[clustering.labels != -1])) mask = clustering.labels == biggest_cluster # mask = clustering.labels != -1 xyt = event.photon_stream.point_cloud x, y, t = xyt.T cleaned_pix = np.zeros(len(image), dtype=bool) k = 0 cleaned_img = np.zeros(len(image)) for i in range(len(lol)): for j in range(len(lol[i])): k += 1 if mask[k - 1]: cleaned_pix[i] = True cleaned_img[i] += 1 cleaned_pix_perc = np.zeros(1440, dtype=bool) for i in range(1440): if cleaned_pix[i] and (cleaned_img[i] > mask.sum() / 200): cleaned_pix_perc[i] = True df = calc_hillas_features_phs(event.photon_stream, clustering) # ell = Ellipse( # [df['cog_x'], df['cog_y']], # df['length']*2, # df['width']*2, # angle=np.rad2deg(df['delta']), # fill=False, linewidth=2, color='b' # ) # ax.add_patch(ell) ell = Ellipse([df['cog_x'], df['cog_y']], df['length'] * 4, df['width'] * 4, angle=np.rad2deg(df['delta']), fill=False, linewidth=1.5, color='b') # ax.add_patch(ell) if is_simulation_event(event): fig.suptitle('run {} event {} reuse {}'.format( event.simulation_truth.run, event.simulation_truth.event, event.simulation_truth.reuse)) else: fig.suptitle('{} event {} delta {:.2f}'.format( file, event.observation_info.event, np.rad2deg(df['delta']))) if feat == 'arrival_times': with warnings.catch_warnings(): warnings.simplefilter("ignore") x = arrival_times - np.nanmean(arrival_times) c = camera(x, cmap='viridis', ax=ax) mark_pixel(cleaned_pix, color=(128 / 255, 186 / 255, 38 / 255), linewidth=2.5) else: c = camera(image, cmap='viridis', ax=ax) mark_pixel(cleaned_pix, color=(128 / 255, 186 / 255, 38 / 255), linewidth=2.5) mark_pixel(cleaned_pix_perc, color='red', linewidth=1.5) ax.axis('off') fig.colorbar(c) pdf.savefig(fig) ax.cla() plt.close(fig) if method == "facttools": print('facttools') with PdfPages('cleaning_facttools_{}_{}.pdf'.format(feat, file)) as pdf: t = Table.read( '/net/big-tank/POOL/projects/fact/photon-stream/facttools/{}/{}_dl1.fits' .format(path, file)) dl2 = read_data( '/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/crab.hdf5', key='events') for i in tqdm(range(number)): fig = plt.figure() ax = fig.add_axes([0.05, 0.05, 0.9, 0.9]) # if path != 'crab': # fig.suptitle('run {} event {} reuse {}'.format(file, t[i]['MCorsikaEvtHeader.fEvtNumber'], t[i]['MCorsikaEvtHeader.fNumReuse'])) # else: # # fig.suptitle('{} event {} delta {:.4f}'.format(file, t[i]['EventNum'], dl2.query('night == 20131104 & run_id == 162 & event_num == {}'.format(t[i]['EventNum']))['delta'].values[0])) t[i]['photoncharge'][t[i]['photoncharge'] < 0] = 0.0 if feat == 'arrival_times': c = camera(t[i]['arrivalTime'] - t[i]['arrivalTime'].mean(), cmap='Spectral', ax=ax) # mark_pixel(t[i]['shower'], color='k', linewidth=2.5) ax.axis('off') cb = fig.colorbar(c) cb.set_label(label=r'$t-\bar{t}$ / ns', fontsize=16) else: c = camera(t[i]['photoncharge'], cmap='viridis', ax=ax) ax.axis('off') cb = fig.colorbar(c) cb.set_label(label=r'Number of Photons', fontsize=16) #mark_pixel(t[i]['shower'], color=(128/255, 186/255, 38/255), linewidth=2.5) # mark_pixel(t[i]['shower'], color=(128/255, 186/255, 38/255), linewidth=2.5) pdf.savefig(fig) ax.cla() plt.close(fig)