def calculate_one_run(inpath, outpath):
hough_responses = []
run = ps.EventListReader(inpath)
number_muons = 0
for event in run:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
cherenkov_point_cloud = photon_clusters.point_cloud
cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask]
point_positions = cherenkov_clusters[:, 0:2]
muon_props = detection(event, photon_clusters)
if muon_props["is_muon"]:
cx = muon_props["muon_ring_cx"]
cy = muon_props["muon_ring_cy"]
r = muon_props["muon_ring_r"]
total_amplitude = evaluate_ring(point_positions, cx, cy, r)
hough_responses.append(total_amplitude)
number_muons += 1
hough_responses = np.multiply(hough_responses, 100)
psf_values = calculate_PSF(hough_responses)
psf_error = psf_values * 1 / np.sqrt(number_muons)
average_psf = float(np.average(psf_values))
outdir = os.path.dirname(outpath)
os.makedirs(outdir, exist_ok=True)
with open(outpath + ".temp", "wt") as fout:
out = {
"average_psf": float(np.average(psf_values)),
"psf_stdev": float(np.std(psf_values)),
"standard_error": np.average(psf_error),
"number_muons": number_muons
}
fout.write(json.dumps(out))
os.rename(outpath + ".temp", outpath)
return 0
def run_job(inpath, outpath, method=False):
results = []
run = ps.EventListReader(inpath)
number_muons = 0
for event in run:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
cherenkov_point_cloud = photon_clusters.point_cloud
cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask]
point_positions = cherenkov_clusters[:, 0:2]
random_state = np.random.get_state()
np.random.seed(event.photon_stream.number_photons)
if not callable(method):
muon_props = extraction.detection(event, photon_clusters)
else:
muon_props = method(event, photon_clusters)
muon_props = extraction.detection(event, photon_clusters)
np.random.set_state(random_state)
if muon_props["is_muon"]:
cx = muon_props["muon_ring_cx"]
cy = muon_props["muon_ring_cy"]
r = muon_props["muon_ring_r"]
total_amplitude = evaluate_ring(point_positions, cx, cy, r)
results.append(total_amplitude)
number_muons += 1
outdir = os.path.dirname(outpath)
os.makedirs(outdir, exist_ok=True)
with open(outpath + ".temp", "wt") as fout:
out = {
"average_fuzz": float(np.average(results)),
"std_fuzz": float(np.std(results)),
"number_muons": number_muons,
}
fout.write(json.dumps(out))
os.rename(outpath + ".temp", outpath)
def run_job(inpath, outpath, method=False):
results = []
run = ps.EventListReader(inpath)
number_muons = 0
for event in run:
clusters = ps.PhotonStreamCluster(event.photon_stream)
random_state = np.random.get_state()
np.random.seed(event.photon_stream.number_photons)
if not callable(method):
muon_props = extraction.detection(event, clusters)
else:
muon_props = method(event, clusters)
np.random.set_state(random_state)
if muon_props["is_muon"]:
std_photons_on_ring = muon_ring_std_event(clusters, muon_props)
results.append(std_photons_on_ring)
number_muons += 1
outdir = os.path.dirname(outpath)
os.makedirs(outdir, exist_ok=True)
with open(outpath + ".temp", "wt") as fout:
out = {
"average_fuzz": float(np.average(results)),
"std_fuzz": float(np.std(results)),
"number_muons": number_muons,
}
fout.write(json.dumps(out))
os.rename(outpath + ".temp", outpath)
def rrr(event):
cluster = ps.PhotonStreamCluster(event.photon_stream)
mask = cluster.labels >= 0
number_photons = mask.sum()
raw_phs = np.zeros(
number_photons + ps.io.magic_constants.NUMBER_OF_PIXELS,
dtype=np.uint8,
)
raw_phs = ps.representations.masked_raw_phs(mask, event.photon_stream.raw)
raw_phsz = raw_phs_to_raw_phs_gz(raw_phs)
evt = {}
evt['raw_phs_gz'] = raw_phsz
evt['cluster'] = cluster
# Air-Shower features
evt['number_photons'] = number_photons
ellipse = features.extract_ellipse(cluster.xyt[mask])
evt['ellipse_cog_x'] = ellipse['center'][0]
evt['ellipse_cog_y'] = ellipse['center'][1]
evt['ellipse_ev0_x'] = ellipse['ev0'][0]
evt['ellipse_ev0_y'] = ellipse['ev0'][1]
evt['ellipse_std0'] = ellipse['std0']
evt['ellipse_std1'] = ellipse['std1']
return evt
def analysis_main(self):
run = ps.EventListReader(self.simulationFile)
ringModel_event_infos = []
medianR_event_infos = []
knownC_event_infos = []
hough_event_infos = []
for event_id, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
hough_muonFeatures = detection(event, photon_clusters)
ringM_muonFeatures = dwsrf(event, photon_clusters)
if hough_muonFeatures["is_muon"]:
hough_event_info = self.extract_with_hough(
event_id, hough_muonFeatures)
hough_event_infos.append(hough_event_info)
if ringM_muonFeatures["is_muon"]:
ringModel_event_info = self.only_ringModel(
event_id, ringM_muonFeatures)
ringModel_event_infos.append(ringModel_event_info)
medianR_event_info = self.medianR_extraction(
ringM_muonFeatures, photon_clusters, event_id)
medianR_event_infos.append(medianR_event_info)
knownC_event_info = self.known_center(event_id,
photon_clusters)
knownC_event_infos.append(knownC_event_info)
data_to_be_saved = [
ringModel_event_infos, medianR_event_infos, knownC_event_infos,
hough_event_infos
]
methods = ["ringM", "medianR", "knownC", "hough"]
for method, data in zip(methods, data_to_be_saved):
self.save_to_file(data, method)
def from_simulation(phs_path, mmcs_corsika_path=None):
event_list = ps.SimulationReader(phs_path,
mmcs_corsika_path=mmcs_corsika_path)
features = []
for event in event_list:
cluster = ps.PhotonStreamCluster(event.photon_stream)
cluster = reject.early_or_late_clusters(cluster)
f = raw_features(photon_stream=event.photon_stream, cluster=cluster)
f['type'] = ps.io.binary.SIMULATION_EVENT_TYPE_KEY
f['az'] = np.deg2rad(event.az)
f['zd'] = np.deg2rad(event.zd)
f['run'] = event.simulation_truth.run
f['event'] = event.simulation_truth.event
f['reuse'] = event.simulation_truth.reuse
f['particle'] = event.simulation_truth.air_shower.particle
f['energy'] = event.simulation_truth.air_shower.energy
f['theta'] = event.simulation_truth.air_shower.theta
f['phi'] = event.simulation_truth.air_shower.phi
f['impact_x'] = event.simulation_truth.air_shower.impact_x(
event.simulation_truth.reuse)
f['impact_y'] = event.simulation_truth.air_shower.impact_y(
event.simulation_truth.reuse)
f['starting_altitude'] = event.simulation_truth.air_shower.starting_altitude
f['hight_of_first_interaction'] = event.simulation_truth.air_shower.hight_of_first_interaction
features.append(f)
triggered = pd.DataFrame(features)
triggered = tools.reduce_DataFrame_to_32_bit(triggered)
thrown = pd.DataFrame(event_list.thrown_events())
thrown = tools.reduce_DataFrame_to_32_bit(thrown)
return triggered, thrown
def extract_muons_from_run(self):
run = ps.EventListReader(self.simulationFile)
event_infos = []
for i, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_features = detection(event, photon_clusters)
if muon_features["is_muon"]:
event_id = i
muon_ring_cx = muon_features['muon_ring_cx']
muon_ring_cy = muon_features['muon_ring_cy']
muon_ring_r = muon_features['muon_ring_r']
mean_arrival_time_muon_cluster = muon_features[
'mean_arrival_time_muon_cluster']
muon_ring_overlapp_with_field_of_view = muon_features[
'muon_ring_overlapp_with_field_of_view']
number_of_photons = muon_features['number_of_photons']
event_info = [
event_id, muon_ring_cx, muon_ring_cy, muon_ring_r,
mean_arrival_time_muon_cluster,
muon_ring_overlapp_with_field_of_view, number_of_photons
]
header = list([
"event_id", "muon_ring_cx", "muon_ring_cy", "muon_ring_r",
"mean_arrival_time_muon_cluster",
"muon_ring_overlapp_with_field_of_view",
"number_of_photons"
])
headers = ",".join(header)
event_infos.append(event_info)
np.savetxt(self.extracted_muons,
event_infos,
delimiter=",",
comments='',
header=headers)
def extract_fuzzParam_from_single_run(self, inpath):
response_resultsR = []
response_resultsH = []
fuzz_resultsR = []
fuzz_resultsH = []
number_muonsH = 0
number_muonsR = 0
mu_event_ids = []
reconstructed_muon_eventsH = []
reconstructed_muon_eventsR = []
run = ps.EventListReader(inpath)
for event_id, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_propsR = ringM(event, photon_clusters)
muon_propsH = hough(event, photon_clusters)
if muon_propsH['is_muon']:
houghResults = self.extraction(muon_propsH, photon_clusters,
event_id)
reconstructed_muon_eventsH.append(houghResults[2])
response_resultsH.append(houghResults[0])
fuzz_resultsH.append(houghResults[1])
number_muonsH += 1
if muon_propsR['is_muon']:
ringM_results = self.extraction(muon_propsR, photon_clusters,
event_id)
reconstructed_muon_eventsR.append(ringM_results[2])
response_resultsR.append(ringM_results[0])
fuzz_resultsR.append(ringM_results[1])
number_muonsR += 1
psf = self.get_point_spread_function(inpath)
responseR_avg, responseR_stdev = self.calculate_average_and_stdev(
response_resultsR)
fuzzR_avg, fuzzR_stdev = self.calculate_average_and_stdev(
fuzz_resultsR)
responseH_avg, responseH_stdev = self.calculate_average_and_stdev(
response_resultsH)
fuzzH_avg, fuzzH_stdev = self.calculate_average_and_stdev(
fuzz_resultsH)
runInfo = {
"reconstructed_muon_eventsH": reconstructed_muon_eventsH,
"responseR_avg": responseR_avg,
"responseR_stdev": responseR_stdev,
"fuzzR_avg": fuzzR_avg,
"fuzzR_stdev": fuzzR_stdev,
"number_muonsH": number_muonsH,
"reconstructed_muon_eventsR": reconstructed_muon_eventsR,
"responseH_avg": responseH_avg,
"responseH_stdev": responseH_stdev,
"fuzzH_avg": fuzzH_avg,
"fuzzH_stdev": fuzzH_stdev,
"number_muonsR": number_muonsR,
"point_spread_function": psf,
"inpath": inpath
}
return runInfo
def plot_time(lol2, event):
lol = event.photon_stream.list_of_lists
clustering = ps.PhotonStreamCluster(event.photon_stream)
if clustering.number > 0:
biggest_cluster = np.argmax(np.bincount(clustering.labels[clustering.labels != -1]))
mask = clustering.labels == biggest_cluster
if mask.sum() > size_cut:
for i in range(len(lol)):
if len(lol[i]) > photons_per_pixel:
for k in range(len(lol[i])):
lol2.append(lol[i][k])
def run_detection(self, inpath):
run = ps.EventListReader(inpath)
path = os.path.normpath(inpath)
split_path = path.split(os.sep)
oa_name = split_path[-2]
oa = re.split('_', oa_name)[2]
preferences = self.read_preferencesFile()
number_of_muons = preferences['--number_of_muons']
found_muons = 0
for event in run:
clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_props = detection(event, clusters)
if muon_props["is_muon"]:
found_muons += 1
return oa, found_muons, number_of_muons
def test_muon_detection():
np.random.seed(seed=1)
muon_truth_path = pkg_resources.resource_filename(
'muons',
os.path.join('tests', 'resources', 'muon_sample_20140101_104.csv'))
muon_truth = np.genfromtxt(muon_truth_path)
muon_sample_path = pkg_resources.resource_filename(
'muons',
os.path.join('tests', 'resources',
'20140101_104_muon_sample.phs.jsonl.gz'))
run = ps.EventListReader(muon_sample_path)
true_positives = 0
true_negatives = 0
false_positives = 0
false_negatives = 0
for event in run:
clusters = ps.PhotonStreamCluster(event.photon_stream)
ret = muons.detection(event, clusters)
if ret['is_muon']:
if event.observation_info.event in muon_truth:
true_positives += 1
else:
false_positives += 1
else:
if event.observation_info.event in muon_truth:
false_negatives += 1
else:
true_negatives += 1
precision = true_positives / (true_positives + false_positives)
sensitivity = true_positives / (true_positives + false_negatives)
print('precision', precision)
print('sensitivity', sensitivity)
assert precision > 0.995
assert sensitivity > 0.76
def test_extraction():
Analysis = pfa.PSF_FuzzAnalysis(
preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir)
simulationFile = os.path.join(
fileDir, "resources", "100simulations_psf0.0.sim.phs")
run = ps.EventListReader(simulationFile)
event_id = 3
for i, event in enumerate(run):
if i == event_id:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_props = {
"muon_ring_cx": 0,
"muon_ring_cy": 0,
"muon_ring_r": 1,
"mean_arrival_time_muon_cluster": 2,
"muon_ring_overlapp_with_field_of_view": 3,
"number_of_photons": 3
}
returns = Analysis.extraction(
muon_props, photon_clusters, event_id)
assert len(returns) == 3
def test_get_fuzziness_parameters():
Analysis = pfa.PSF_FuzzAnalysis(
preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir)
simulationFile = os.path.join(
fileDir, "resources", "100simulations_psf0.0.sim.phs")
run = ps.EventListReader(simulationFile)
for i, event in enumerate(run):
if i == 47:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_props = {
"muon_ring_cx": 0,
"muon_ring_cy": 0,
"muon_ring_r": 1,
"mean_arrival_time_muon_cluster": 2,
"muon_ring_overlapp_with_field_of_view": 3,
"number_of_photons": 3
}
normed_response, fuzziness_stdParam = Analysis.get_fuzziness_parameters(
photon_clusters, muon_props)
assert (
isinstance(normed_response, Number) and
isinstance(fuzziness_stdParam, Number)
)
def do_clustering(self, pure_cherenkov_events_path, events_with_nsb_path):
pure_cherenkov_run = ps.EventListReader(pure_cherenkov_events_path)
nsb_run = ps.EventListReader(events_with_nsb_path)
nsb_run_found_photons = []
pure_cherenkov_run_photons = []
cut_cherenkov = []
all_photons_run = []
for event in nsb_run:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
nsb_cherenkov_photon_stream = photon_clusters.point_cloud
nsb_cherenkov_ps = nsb_cherenkov_photon_stream[
cherenkov_cluster_mask]
cherenkov_photons = self.cut_hist(nsb_cherenkov_ps)
cut_cherenkov.append(cherenkov_photons)
nsb_run_found_photons.append(nsb_cherenkov_ps[:, 0:3])
all_photons = event.photon_stream.point_cloud
all_photons_run.append(all_photons)
for muon in pure_cherenkov_run:
pure_photon_stream = muon.photon_stream.point_cloud
pure_cherenkov_run_photons.append(pure_photon_stream)
return (all_photons_run, pure_cherenkov_run_photons,
nsb_run_found_photons, cut_cherenkov)
def extract_single_simulation_features(event, cluster=None, min_samples=20):
"""
Extracts features from a single PHS simulation events and returns them
:param phs_event: PHS simulation event
:return:
"""
if cluster is None:
cluster = phs.PhotonStreamCluster(event.photon_stream, min_samples=min_samples)
cluster = reject.early_or_late_clusters(cluster)
features = phs_analysis.extract.raw_features(
photon_stream=event.photon_stream, cluster=cluster
)
features["type"] = phs.io.binary.SIMULATION_EVENT_TYPE_KEY
features["az"] = np.deg2rad(event.az)
features["zd"] = np.deg2rad(event.zd)
features["run"] = event.simulation_truth.run
features["event"] = event.simulation_truth.event
features["reuse"] = event.simulation_truth.reuse
features["particle"] = event.simulation_truth.air_shower.particle
features["energy"] = event.simulation_truth.air_shower.energy
features["theta"] = event.simulation_truth.air_shower.theta
features["phi"] = event.simulation_truth.air_shower.phi
features["impact_x"] = event.simulation_truth.air_shower.impact_x(
event.simulation_truth.reuse
)
features["impact_y"] = event.simulation_truth.air_shower.impact_y(
event.simulation_truth.reuse
)
features["starting_altitude"] = event.simulation_truth.air_shower.starting_altitude
features[
"height_of_first_interaction"
] = event.simulation_truth.air_shower.height_of_first_interaction
return features, cluster
def extract_single_observation_features(event):
"""
Extracts features from a single PHS observation event and returns them
:param phs_event: PHS observation event
:return:
"""
cluster = phs.PhotonStreamCluster(event.photon_stream)
cluster = reject.early_or_late_clusters(cluster)
features = phs_analysis.extract.raw_features(
photon_stream=event.photon_stream, cluster=cluster
)
features["type"] = phs.io.binary.OBSERVATION_EVENT_TYPE_KEY
features["az"] = np.deg2rad(event.az)
features["zd"] = np.deg2rad(event.zd)
features["night"] = event.observation_info.night
features["run"] = event.observation_info.run
features["event"] = event.observation_info.event
features["time"] = (
event.observation_info._time_unix_s + event.observation_info._time_unix_us / 1e6
)
return features, cluster
def from_observation(phs_path):
event_list = ps.EventListReader(phs_path)
features = []
for event in event_list:
if event.observation_info.trigger_type == PHYSICS_TRIGGER:
cluster = ps.PhotonStreamCluster(event.photon_stream)
cluster = reject.early_or_late_clusters(cluster)
f = raw_features(photon_stream=event.photon_stream,
cluster=cluster)
f['type'] = ps.io.binary.OBSERVATION_EVENT_TYPE_KEY
f['az'] = np.deg2rad(event.az)
f['zd'] = np.deg2rad(event.zd)
f['night'] = event.observation_info.night
f['run'] = event.observation_info.run
f['event'] = event.observation_info.event
f['time'] = event.observation_info._time_unix_s + event.observation_info._time_unix_us / 1e6
features.append(f)
triggered = pd.DataFrame(features)
triggered = tools.reduce_DataFrame_to_32_bit(triggered)
return triggered
def run_job(self, job):
results = []
inpath = job["inpath"]
output_path_responseH = job["output_path_responseH"]
output_path_stdevR = job["output_path_stdevR"]
output_path_responseR = job["output_path_responseR"]
output_path_stdevH = job["output_path_stdevH"]
muonCountH = 0
muonCountR = 0
run = ps.EventListReader(inpath)
muon_ring_featuresR = []
muon_ring_featuresH = []
fuzziness_stdParamRs = []
fuzziness_stdParamHs = []
normed_responseRs = []
normed_responseHs = []
fuzz_paramsH = []
for event_id, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_propsR = ringM_detection(event, photon_clusters)
muon_propsH = detection(event, photon_clusters)
if muon_propsH["is_muon"]:
muonCountH += 1
normed_responseH, fuzziness_stdParamH = (
self.get_fuzziness_parameters(photon_clusters,
muon_propsH))
muon_ring_featureH = [
muon_propsH["muon_ring_cx"], muon_propsH["muon_ring_cy"],
muon_propsH["muon_ring_r"]
]
muon_ring_featuresH.append(muon_ring_featureH)
fuzziness_stdParamHs.append(fuzziness_stdParamH)
normed_responseHs.append(normed_responseH)
if muon_propsR["is_muon"]:
muonCountR += 1
normed_responseR, fuzziness_stdParamR = (
self.get_fuzziness_parameters(photon_clusters,
muon_propsR))
muon_ring_featureR = [
muon_propsR["muon_ring_cx"], muon_propsR["muon_ring_cy"],
muon_propsR["muon_ring_r"]
]
fuzziness_stdParamRs.append(fuzziness_stdParamR)
normed_responseRs.append(normed_responseR)
muon_ring_featuresR.append(muon_ring_featureR)
fact_path = fact.path.parse(inpath)
night = fact_path["night"]
run = fact_path["run"]
filename = str(night) + "_" + str(run) + ".csv"
output_dirR = os.path.dirname(output_path_stdevR)
output_dirH = os.path.dirname(output_path_stdevH)
if not os.path.isdir(output_dirR):
os.makedirs(output_dirR, exist_ok=True)
if not os.path.isdir(output_dirH):
os.makedirs(output_dirH, exist_ok=True)
self.save_to_file("ringM", output_dirR, filename, muon_ring_featuresR)
self.save_to_file("hough", output_dirH, filename, muon_ring_featuresH)
self.save_fuzz_param(output_path_stdevR, fuzziness_stdParamRs,
muonCountR)
self.save_fuzz_param(output_path_responseR, normed_responseRs,
muonCountR)
self.save_fuzz_param(output_path_stdevH, fuzziness_stdParamHs,
muonCountH)
self.save_fuzz_param(output_path_responseH, normed_responseHs,
muonCountH)
return 0
def extract_muons_from_run(
input_run_path,
output_run_path,
output_run_header_path
):
"""
Detects and extracts muon candidate events from a run. The muon candidate
events are exported into a new output run. In addidion a header for the
muon candidates is exported.
Parameter
---------
input_run_path Path to the input run.
output_run_path Path to the output run of muon candidates.
output_run_header_path Path to the binary output run header.
Binary Output Format Run Header
-------------------------------
for each muon candidate:
1) uint32 Night
2) uint32 Run ID
3) uint32 Event ID
4) uint32 unix time seconds [s]
5) uint32 unix time micro seconds modulo full seconds [us]
6) float32 Pointing zenith distance [deg]
7) float32 Pointing azimuth [deg]
8) float32 muon ring center x [deg]
9) float32 muon ring center y [deg]
10) float32 muon ring radius [deg]
11) float32 mean arrival time muon cluster [s]
12) float32 muon ring overlapp with field of view (0.0 to 1.0) [1]
13) float32 number of photons muon cluster [1]
"""
run = ps.EventListReader(input_run_path)
with gzip.open(output_run_path, 'wt') as f_muon_run, \
open(output_run_header_path, 'wb') as f_muon_run_header:
for event in run:
if (
event.observation_info.trigger_type ==
FACT_PHYSICS_SELF_TRIGGER
):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_features = detection(event, photon_clusters)
if muon_features['is_muon']:
# EXPORT EVENT in JSON
event_dict = ps.io.jsonl.event_to_dict(event)
json.dump(event_dict, f_muon_run)
f_muon_run.write('\n')
# EXPORT EVENT header
head1 = np.zeros(5, dtype=np.uint32)
head1[0] = event.observation_info.night
head1[1] = event.observation_info.run
head1[2] = event.observation_info.event
head1[3] = event.observation_info._time_unix_s
head1[4] = event.observation_info._time_unix_us
head2 = np.zeros(8, dtype=np.float32)
head2[0] = event.zd
head2[1] = event.az
head2[2] = muon_features['muon_ring_cx']*rad2deg
head2[3] = muon_features['muon_ring_cy']*rad2deg
head2[4] = muon_features['muon_ring_r']*rad2deg
head2[5] = muon_features['mean_arrival_time_muon_cluster']
head2[6] = muon_features[
'muon_ring_overlapp_with_field_of_view'
]
head2[7] = muon_features['number_of_photons']
f_muon_run_header.write(head1.tobytes())
f_muon_run_header.write(head2.tobytes())
