def calculate_one_run(inpath, outpath): hough_responses = [] run = ps.EventListReader(inpath) number_muons = 0 for event in run: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 cherenkov_point_cloud = photon_clusters.point_cloud cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask] point_positions = cherenkov_clusters[:, 0:2] muon_props = detection(event, photon_clusters) if muon_props["is_muon"]: cx = muon_props["muon_ring_cx"] cy = muon_props["muon_ring_cy"] r = muon_props["muon_ring_r"] total_amplitude = evaluate_ring(point_positions, cx, cy, r) hough_responses.append(total_amplitude) number_muons += 1 hough_responses = np.multiply(hough_responses, 100) psf_values = calculate_PSF(hough_responses) psf_error = psf_values * 1 / np.sqrt(number_muons) average_psf = float(np.average(psf_values)) outdir = os.path.dirname(outpath) os.makedirs(outdir, exist_ok=True) with open(outpath + ".temp", "wt") as fout: out = { "average_psf": float(np.average(psf_values)), "psf_stdev": float(np.std(psf_values)), "standard_error": np.average(psf_error), "number_muons": number_muons } fout.write(json.dumps(out)) os.rename(outpath + ".temp", outpath) return 0
def run_job(inpath, outpath, method=False): results = [] run = ps.EventListReader(inpath) number_muons = 0 for event in run: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 cherenkov_point_cloud = photon_clusters.point_cloud cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask] point_positions = cherenkov_clusters[:, 0:2] random_state = np.random.get_state() np.random.seed(event.photon_stream.number_photons) if not callable(method): muon_props = extraction.detection(event, photon_clusters) else: muon_props = method(event, photon_clusters) muon_props = extraction.detection(event, photon_clusters) np.random.set_state(random_state) if muon_props["is_muon"]: cx = muon_props["muon_ring_cx"] cy = muon_props["muon_ring_cy"] r = muon_props["muon_ring_r"] total_amplitude = evaluate_ring(point_positions, cx, cy, r) results.append(total_amplitude) number_muons += 1 outdir = os.path.dirname(outpath) os.makedirs(outdir, exist_ok=True) with open(outpath + ".temp", "wt") as fout: out = { "average_fuzz": float(np.average(results)), "std_fuzz": float(np.std(results)), "number_muons": number_muons, } fout.write(json.dumps(out)) os.rename(outpath + ".temp", outpath)
def run_job(inpath, outpath, method=False): results = [] run = ps.EventListReader(inpath) number_muons = 0 for event in run: clusters = ps.PhotonStreamCluster(event.photon_stream) random_state = np.random.get_state() np.random.seed(event.photon_stream.number_photons) if not callable(method): muon_props = extraction.detection(event, clusters) else: muon_props = method(event, clusters) np.random.set_state(random_state) if muon_props["is_muon"]: std_photons_on_ring = muon_ring_std_event(clusters, muon_props) results.append(std_photons_on_ring) number_muons += 1 outdir = os.path.dirname(outpath) os.makedirs(outdir, exist_ok=True) with open(outpath + ".temp", "wt") as fout: out = { "average_fuzz": float(np.average(results)), "std_fuzz": float(np.std(results)), "number_muons": number_muons, } fout.write(json.dumps(out)) os.rename(outpath + ".temp", outpath)
def rrr(event): cluster = ps.PhotonStreamCluster(event.photon_stream) mask = cluster.labels >= 0 number_photons = mask.sum() raw_phs = np.zeros( number_photons + ps.io.magic_constants.NUMBER_OF_PIXELS, dtype=np.uint8, ) raw_phs = ps.representations.masked_raw_phs(mask, event.photon_stream.raw) raw_phsz = raw_phs_to_raw_phs_gz(raw_phs) evt = {} evt['raw_phs_gz'] = raw_phsz evt['cluster'] = cluster # Air-Shower features evt['number_photons'] = number_photons ellipse = features.extract_ellipse(cluster.xyt[mask]) evt['ellipse_cog_x'] = ellipse['center'][0] evt['ellipse_cog_y'] = ellipse['center'][1] evt['ellipse_ev0_x'] = ellipse['ev0'][0] evt['ellipse_ev0_y'] = ellipse['ev0'][1] evt['ellipse_std0'] = ellipse['std0'] evt['ellipse_std1'] = ellipse['std1'] return evt
def analysis_main(self): run = ps.EventListReader(self.simulationFile) ringModel_event_infos = [] medianR_event_infos = [] knownC_event_infos = [] hough_event_infos = [] for event_id, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) hough_muonFeatures = detection(event, photon_clusters) ringM_muonFeatures = dwsrf(event, photon_clusters) if hough_muonFeatures["is_muon"]: hough_event_info = self.extract_with_hough( event_id, hough_muonFeatures) hough_event_infos.append(hough_event_info) if ringM_muonFeatures["is_muon"]: ringModel_event_info = self.only_ringModel( event_id, ringM_muonFeatures) ringModel_event_infos.append(ringModel_event_info) medianR_event_info = self.medianR_extraction( ringM_muonFeatures, photon_clusters, event_id) medianR_event_infos.append(medianR_event_info) knownC_event_info = self.known_center(event_id, photon_clusters) knownC_event_infos.append(knownC_event_info) data_to_be_saved = [ ringModel_event_infos, medianR_event_infos, knownC_event_infos, hough_event_infos ] methods = ["ringM", "medianR", "knownC", "hough"] for method, data in zip(methods, data_to_be_saved): self.save_to_file(data, method)
def from_simulation(phs_path, mmcs_corsika_path=None): event_list = ps.SimulationReader(phs_path, mmcs_corsika_path=mmcs_corsika_path) features = [] for event in event_list: cluster = ps.PhotonStreamCluster(event.photon_stream) cluster = reject.early_or_late_clusters(cluster) f = raw_features(photon_stream=event.photon_stream, cluster=cluster) f['type'] = ps.io.binary.SIMULATION_EVENT_TYPE_KEY f['az'] = np.deg2rad(event.az) f['zd'] = np.deg2rad(event.zd) f['run'] = event.simulation_truth.run f['event'] = event.simulation_truth.event f['reuse'] = event.simulation_truth.reuse f['particle'] = event.simulation_truth.air_shower.particle f['energy'] = event.simulation_truth.air_shower.energy f['theta'] = event.simulation_truth.air_shower.theta f['phi'] = event.simulation_truth.air_shower.phi f['impact_x'] = event.simulation_truth.air_shower.impact_x( event.simulation_truth.reuse) f['impact_y'] = event.simulation_truth.air_shower.impact_y( event.simulation_truth.reuse) f['starting_altitude'] = event.simulation_truth.air_shower.starting_altitude f['hight_of_first_interaction'] = event.simulation_truth.air_shower.hight_of_first_interaction features.append(f) triggered = pd.DataFrame(features) triggered = tools.reduce_DataFrame_to_32_bit(triggered) thrown = pd.DataFrame(event_list.thrown_events()) thrown = tools.reduce_DataFrame_to_32_bit(thrown) return triggered, thrown
def extract_muons_from_run(self): run = ps.EventListReader(self.simulationFile) event_infos = [] for i, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_features = detection(event, photon_clusters) if muon_features["is_muon"]: event_id = i muon_ring_cx = muon_features['muon_ring_cx'] muon_ring_cy = muon_features['muon_ring_cy'] muon_ring_r = muon_features['muon_ring_r'] mean_arrival_time_muon_cluster = muon_features[ 'mean_arrival_time_muon_cluster'] muon_ring_overlapp_with_field_of_view = muon_features[ 'muon_ring_overlapp_with_field_of_view'] number_of_photons = muon_features['number_of_photons'] event_info = [ event_id, muon_ring_cx, muon_ring_cy, muon_ring_r, mean_arrival_time_muon_cluster, muon_ring_overlapp_with_field_of_view, number_of_photons ] header = list([ "event_id", "muon_ring_cx", "muon_ring_cy", "muon_ring_r", "mean_arrival_time_muon_cluster", "muon_ring_overlapp_with_field_of_view", "number_of_photons" ]) headers = ",".join(header) event_infos.append(event_info) np.savetxt(self.extracted_muons, event_infos, delimiter=",", comments='', header=headers)
def extract_fuzzParam_from_single_run(self, inpath): response_resultsR = [] response_resultsH = [] fuzz_resultsR = [] fuzz_resultsH = [] number_muonsH = 0 number_muonsR = 0 mu_event_ids = [] reconstructed_muon_eventsH = [] reconstructed_muon_eventsR = [] run = ps.EventListReader(inpath) for event_id, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_propsR = ringM(event, photon_clusters) muon_propsH = hough(event, photon_clusters) if muon_propsH['is_muon']: houghResults = self.extraction(muon_propsH, photon_clusters, event_id) reconstructed_muon_eventsH.append(houghResults[2]) response_resultsH.append(houghResults[0]) fuzz_resultsH.append(houghResults[1]) number_muonsH += 1 if muon_propsR['is_muon']: ringM_results = self.extraction(muon_propsR, photon_clusters, event_id) reconstructed_muon_eventsR.append(ringM_results[2]) response_resultsR.append(ringM_results[0]) fuzz_resultsR.append(ringM_results[1]) number_muonsR += 1 psf = self.get_point_spread_function(inpath) responseR_avg, responseR_stdev = self.calculate_average_and_stdev( response_resultsR) fuzzR_avg, fuzzR_stdev = self.calculate_average_and_stdev( fuzz_resultsR) responseH_avg, responseH_stdev = self.calculate_average_and_stdev( response_resultsH) fuzzH_avg, fuzzH_stdev = self.calculate_average_and_stdev( fuzz_resultsH) runInfo = { "reconstructed_muon_eventsH": reconstructed_muon_eventsH, "responseR_avg": responseR_avg, "responseR_stdev": responseR_stdev, "fuzzR_avg": fuzzR_avg, "fuzzR_stdev": fuzzR_stdev, "number_muonsH": number_muonsH, "reconstructed_muon_eventsR": reconstructed_muon_eventsR, "responseH_avg": responseH_avg, "responseH_stdev": responseH_stdev, "fuzzH_avg": fuzzH_avg, "fuzzH_stdev": fuzzH_stdev, "number_muonsR": number_muonsR, "point_spread_function": psf, "inpath": inpath } return runInfo
def plot_time(lol2, event): lol = event.photon_stream.list_of_lists clustering = ps.PhotonStreamCluster(event.photon_stream) if clustering.number > 0: biggest_cluster = np.argmax(np.bincount(clustering.labels[clustering.labels != -1])) mask = clustering.labels == biggest_cluster if mask.sum() > size_cut: for i in range(len(lol)): if len(lol[i]) > photons_per_pixel: for k in range(len(lol[i])): lol2.append(lol[i][k])
def run_detection(self, inpath): run = ps.EventListReader(inpath) path = os.path.normpath(inpath) split_path = path.split(os.sep) oa_name = split_path[-2] oa = re.split('_', oa_name)[2] preferences = self.read_preferencesFile() number_of_muons = preferences['--number_of_muons'] found_muons = 0 for event in run: clusters = ps.PhotonStreamCluster(event.photon_stream) muon_props = detection(event, clusters) if muon_props["is_muon"]: found_muons += 1 return oa, found_muons, number_of_muons
def test_muon_detection(): np.random.seed(seed=1) muon_truth_path = pkg_resources.resource_filename( 'muons', os.path.join('tests', 'resources', 'muon_sample_20140101_104.csv')) muon_truth = np.genfromtxt(muon_truth_path) muon_sample_path = pkg_resources.resource_filename( 'muons', os.path.join('tests', 'resources', '20140101_104_muon_sample.phs.jsonl.gz')) run = ps.EventListReader(muon_sample_path) true_positives = 0 true_negatives = 0 false_positives = 0 false_negatives = 0 for event in run: clusters = ps.PhotonStreamCluster(event.photon_stream) ret = muons.detection(event, clusters) if ret['is_muon']: if event.observation_info.event in muon_truth: true_positives += 1 else: false_positives += 1 else: if event.observation_info.event in muon_truth: false_negatives += 1 else: true_negatives += 1 precision = true_positives / (true_positives + false_positives) sensitivity = true_positives / (true_positives + false_negatives) print('precision', precision) print('sensitivity', sensitivity) assert precision > 0.995 assert sensitivity > 0.76
def test_extraction(): Analysis = pfa.PSF_FuzzAnalysis( preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir) simulationFile = os.path.join( fileDir, "resources", "100simulations_psf0.0.sim.phs") run = ps.EventListReader(simulationFile) event_id = 3 for i, event in enumerate(run): if i == event_id: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_props = { "muon_ring_cx": 0, "muon_ring_cy": 0, "muon_ring_r": 1, "mean_arrival_time_muon_cluster": 2, "muon_ring_overlapp_with_field_of_view": 3, "number_of_photons": 3 } returns = Analysis.extraction( muon_props, photon_clusters, event_id) assert len(returns) == 3
def test_get_fuzziness_parameters(): Analysis = pfa.PSF_FuzzAnalysis( preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir) simulationFile = os.path.join( fileDir, "resources", "100simulations_psf0.0.sim.phs") run = ps.EventListReader(simulationFile) for i, event in enumerate(run): if i == 47: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_props = { "muon_ring_cx": 0, "muon_ring_cy": 0, "muon_ring_r": 1, "mean_arrival_time_muon_cluster": 2, "muon_ring_overlapp_with_field_of_view": 3, "number_of_photons": 3 } normed_response, fuzziness_stdParam = Analysis.get_fuzziness_parameters( photon_clusters, muon_props) assert ( isinstance(normed_response, Number) and isinstance(fuzziness_stdParam, Number) )
def do_clustering(self, pure_cherenkov_events_path, events_with_nsb_path): pure_cherenkov_run = ps.EventListReader(pure_cherenkov_events_path) nsb_run = ps.EventListReader(events_with_nsb_path) nsb_run_found_photons = [] pure_cherenkov_run_photons = [] cut_cherenkov = [] all_photons_run = [] for event in nsb_run: photon_clusters = ps.PhotonStreamCluster(event.photon_stream) cherenkov_cluster_mask = photon_clusters.labels >= 0 nsb_cherenkov_photon_stream = photon_clusters.point_cloud nsb_cherenkov_ps = nsb_cherenkov_photon_stream[ cherenkov_cluster_mask] cherenkov_photons = self.cut_hist(nsb_cherenkov_ps) cut_cherenkov.append(cherenkov_photons) nsb_run_found_photons.append(nsb_cherenkov_ps[:, 0:3]) all_photons = event.photon_stream.point_cloud all_photons_run.append(all_photons) for muon in pure_cherenkov_run: pure_photon_stream = muon.photon_stream.point_cloud pure_cherenkov_run_photons.append(pure_photon_stream) return (all_photons_run, pure_cherenkov_run_photons, nsb_run_found_photons, cut_cherenkov)
def extract_single_simulation_features(event, cluster=None, min_samples=20): """ Extracts features from a single PHS simulation events and returns them :param phs_event: PHS simulation event :return: """ if cluster is None: cluster = phs.PhotonStreamCluster(event.photon_stream, min_samples=min_samples) cluster = reject.early_or_late_clusters(cluster) features = phs_analysis.extract.raw_features( photon_stream=event.photon_stream, cluster=cluster ) features["type"] = phs.io.binary.SIMULATION_EVENT_TYPE_KEY features["az"] = np.deg2rad(event.az) features["zd"] = np.deg2rad(event.zd) features["run"] = event.simulation_truth.run features["event"] = event.simulation_truth.event features["reuse"] = event.simulation_truth.reuse features["particle"] = event.simulation_truth.air_shower.particle features["energy"] = event.simulation_truth.air_shower.energy features["theta"] = event.simulation_truth.air_shower.theta features["phi"] = event.simulation_truth.air_shower.phi features["impact_x"] = event.simulation_truth.air_shower.impact_x( event.simulation_truth.reuse ) features["impact_y"] = event.simulation_truth.air_shower.impact_y( event.simulation_truth.reuse ) features["starting_altitude"] = event.simulation_truth.air_shower.starting_altitude features[ "height_of_first_interaction" ] = event.simulation_truth.air_shower.height_of_first_interaction return features, cluster
def extract_single_observation_features(event): """ Extracts features from a single PHS observation event and returns them :param phs_event: PHS observation event :return: """ cluster = phs.PhotonStreamCluster(event.photon_stream) cluster = reject.early_or_late_clusters(cluster) features = phs_analysis.extract.raw_features( photon_stream=event.photon_stream, cluster=cluster ) features["type"] = phs.io.binary.OBSERVATION_EVENT_TYPE_KEY features["az"] = np.deg2rad(event.az) features["zd"] = np.deg2rad(event.zd) features["night"] = event.observation_info.night features["run"] = event.observation_info.run features["event"] = event.observation_info.event features["time"] = ( event.observation_info._time_unix_s + event.observation_info._time_unix_us / 1e6 ) return features, cluster
def from_observation(phs_path): event_list = ps.EventListReader(phs_path) features = [] for event in event_list: if event.observation_info.trigger_type == PHYSICS_TRIGGER: cluster = ps.PhotonStreamCluster(event.photon_stream) cluster = reject.early_or_late_clusters(cluster) f = raw_features(photon_stream=event.photon_stream, cluster=cluster) f['type'] = ps.io.binary.OBSERVATION_EVENT_TYPE_KEY f['az'] = np.deg2rad(event.az) f['zd'] = np.deg2rad(event.zd) f['night'] = event.observation_info.night f['run'] = event.observation_info.run f['event'] = event.observation_info.event f['time'] = event.observation_info._time_unix_s + event.observation_info._time_unix_us / 1e6 features.append(f) triggered = pd.DataFrame(features) triggered = tools.reduce_DataFrame_to_32_bit(triggered) return triggered
def run_job(self, job): results = [] inpath = job["inpath"] output_path_responseH = job["output_path_responseH"] output_path_stdevR = job["output_path_stdevR"] output_path_responseR = job["output_path_responseR"] output_path_stdevH = job["output_path_stdevH"] muonCountH = 0 muonCountR = 0 run = ps.EventListReader(inpath) muon_ring_featuresR = [] muon_ring_featuresH = [] fuzziness_stdParamRs = [] fuzziness_stdParamHs = [] normed_responseRs = [] normed_responseHs = [] fuzz_paramsH = [] for event_id, event in enumerate(run): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_propsR = ringM_detection(event, photon_clusters) muon_propsH = detection(event, photon_clusters) if muon_propsH["is_muon"]: muonCountH += 1 normed_responseH, fuzziness_stdParamH = ( self.get_fuzziness_parameters(photon_clusters, muon_propsH)) muon_ring_featureH = [ muon_propsH["muon_ring_cx"], muon_propsH["muon_ring_cy"], muon_propsH["muon_ring_r"] ] muon_ring_featuresH.append(muon_ring_featureH) fuzziness_stdParamHs.append(fuzziness_stdParamH) normed_responseHs.append(normed_responseH) if muon_propsR["is_muon"]: muonCountR += 1 normed_responseR, fuzziness_stdParamR = ( self.get_fuzziness_parameters(photon_clusters, muon_propsR)) muon_ring_featureR = [ muon_propsR["muon_ring_cx"], muon_propsR["muon_ring_cy"], muon_propsR["muon_ring_r"] ] fuzziness_stdParamRs.append(fuzziness_stdParamR) normed_responseRs.append(normed_responseR) muon_ring_featuresR.append(muon_ring_featureR) fact_path = fact.path.parse(inpath) night = fact_path["night"] run = fact_path["run"] filename = str(night) + "_" + str(run) + ".csv" output_dirR = os.path.dirname(output_path_stdevR) output_dirH = os.path.dirname(output_path_stdevH) if not os.path.isdir(output_dirR): os.makedirs(output_dirR, exist_ok=True) if not os.path.isdir(output_dirH): os.makedirs(output_dirH, exist_ok=True) self.save_to_file("ringM", output_dirR, filename, muon_ring_featuresR) self.save_to_file("hough", output_dirH, filename, muon_ring_featuresH) self.save_fuzz_param(output_path_stdevR, fuzziness_stdParamRs, muonCountR) self.save_fuzz_param(output_path_responseR, normed_responseRs, muonCountR) self.save_fuzz_param(output_path_stdevH, fuzziness_stdParamHs, muonCountH) self.save_fuzz_param(output_path_responseH, normed_responseHs, muonCountH) return 0
def extract_muons_from_run( input_run_path, output_run_path, output_run_header_path ): """ Detects and extracts muon candidate events from a run. The muon candidate events are exported into a new output run. In addidion a header for the muon candidates is exported. Parameter --------- input_run_path Path to the input run. output_run_path Path to the output run of muon candidates. output_run_header_path Path to the binary output run header. Binary Output Format Run Header ------------------------------- for each muon candidate: 1) uint32 Night 2) uint32 Run ID 3) uint32 Event ID 4) uint32 unix time seconds [s] 5) uint32 unix time micro seconds modulo full seconds [us] 6) float32 Pointing zenith distance [deg] 7) float32 Pointing azimuth [deg] 8) float32 muon ring center x [deg] 9) float32 muon ring center y [deg] 10) float32 muon ring radius [deg] 11) float32 mean arrival time muon cluster [s] 12) float32 muon ring overlapp with field of view (0.0 to 1.0) [1] 13) float32 number of photons muon cluster [1] """ run = ps.EventListReader(input_run_path) with gzip.open(output_run_path, 'wt') as f_muon_run, \ open(output_run_header_path, 'wb') as f_muon_run_header: for event in run: if ( event.observation_info.trigger_type == FACT_PHYSICS_SELF_TRIGGER ): photon_clusters = ps.PhotonStreamCluster(event.photon_stream) muon_features = detection(event, photon_clusters) if muon_features['is_muon']: # EXPORT EVENT in JSON event_dict = ps.io.jsonl.event_to_dict(event) json.dump(event_dict, f_muon_run) f_muon_run.write('\n') # EXPORT EVENT header head1 = np.zeros(5, dtype=np.uint32) head1[0] = event.observation_info.night head1[1] = event.observation_info.run head1[2] = event.observation_info.event head1[3] = event.observation_info._time_unix_s head1[4] = event.observation_info._time_unix_us head2 = np.zeros(8, dtype=np.float32) head2[0] = event.zd head2[1] = event.az head2[2] = muon_features['muon_ring_cx']*rad2deg head2[3] = muon_features['muon_ring_cy']*rad2deg head2[4] = muon_features['muon_ring_r']*rad2deg head2[5] = muon_features['mean_arrival_time_muon_cluster'] head2[6] = muon_features[ 'muon_ring_overlapp_with_field_of_view' ] head2[7] = muon_features['number_of_photons'] f_muon_run_header.write(head1.tobytes()) f_muon_run_header.write(head2.tobytes())