def calculate_one_run(inpath, outpath):
hough_responses = []
run = ps.EventListReader(inpath)
number_muons = 0
for event in run:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
cherenkov_point_cloud = photon_clusters.point_cloud
cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask]
point_positions = cherenkov_clusters[:, 0:2]
muon_props = detection(event, photon_clusters)
if muon_props["is_muon"]:
cx = muon_props["muon_ring_cx"]
cy = muon_props["muon_ring_cy"]
r = muon_props["muon_ring_r"]
total_amplitude = evaluate_ring(point_positions, cx, cy, r)
hough_responses.append(total_amplitude)
number_muons += 1
hough_responses = np.multiply(hough_responses, 100)
psf_values = calculate_PSF(hough_responses)
psf_error = psf_values * 1 / np.sqrt(number_muons)
average_psf = float(np.average(psf_values))
outdir = os.path.dirname(outpath)
os.makedirs(outdir, exist_ok=True)
with open(outpath + ".temp", "wt") as fout:
out = {
"average_psf": float(np.average(psf_values)),
"psf_stdev": float(np.std(psf_values)),
"standard_error": np.average(psf_error),
"number_muons": number_muons
}
fout.write(json.dumps(out))
os.rename(outpath + ".temp", outpath)
return 0
def run_job(inpath, outpath, method=False):
results = []
run = ps.EventListReader(inpath)
number_muons = 0
for event in run:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
cherenkov_point_cloud = photon_clusters.point_cloud
cherenkov_clusters = cherenkov_point_cloud[cherenkov_cluster_mask]
point_positions = cherenkov_clusters[:, 0:2]
random_state = np.random.get_state()
np.random.seed(event.photon_stream.number_photons)
if not callable(method):
muon_props = extraction.detection(event, photon_clusters)
else:
muon_props = method(event, photon_clusters)
muon_props = extraction.detection(event, photon_clusters)
np.random.set_state(random_state)
if muon_props["is_muon"]:
cx = muon_props["muon_ring_cx"]
cy = muon_props["muon_ring_cy"]
r = muon_props["muon_ring_r"]
total_amplitude = evaluate_ring(point_positions, cx, cy, r)
results.append(total_amplitude)
number_muons += 1
outdir = os.path.dirname(outpath)
os.makedirs(outdir, exist_ok=True)
with open(outpath + ".temp", "wt") as fout:
out = {
"average_fuzz": float(np.average(results)),
"std_fuzz": float(np.std(results)),
"number_muons": number_muons,
}
fout.write(json.dumps(out))
os.rename(outpath + ".temp", outpath)
def extract_muons_from_run(self):
run = ps.EventListReader(self.simulationFile)
event_infos = []
for i, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_features = detection(event, photon_clusters)
if muon_features["is_muon"]:
event_id = i
muon_ring_cx = muon_features['muon_ring_cx']
muon_ring_cy = muon_features['muon_ring_cy']
muon_ring_r = muon_features['muon_ring_r']
mean_arrival_time_muon_cluster = muon_features[
'mean_arrival_time_muon_cluster']
muon_ring_overlapp_with_field_of_view = muon_features[
'muon_ring_overlapp_with_field_of_view']
number_of_photons = muon_features['number_of_photons']
event_info = [
event_id, muon_ring_cx, muon_ring_cy, muon_ring_r,
mean_arrival_time_muon_cluster,
muon_ring_overlapp_with_field_of_view, number_of_photons
]
header = list([
"event_id", "muon_ring_cx", "muon_ring_cy", "muon_ring_r",
"mean_arrival_time_muon_cluster",
"muon_ring_overlapp_with_field_of_view",
"number_of_photons"
])
headers = ",".join(header)
event_infos.append(event_info)
np.savetxt(self.extracted_muons,
event_infos,
delimiter=",",
comments='',
header=headers)
def analysis_main(self):
run = ps.EventListReader(self.simulationFile)
ringModel_event_infos = []
medianR_event_infos = []
knownC_event_infos = []
hough_event_infos = []
for event_id, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
hough_muonFeatures = detection(event, photon_clusters)
ringM_muonFeatures = dwsrf(event, photon_clusters)
if hough_muonFeatures["is_muon"]:
hough_event_info = self.extract_with_hough(
event_id, hough_muonFeatures)
hough_event_infos.append(hough_event_info)
if ringM_muonFeatures["is_muon"]:
ringModel_event_info = self.only_ringModel(
event_id, ringM_muonFeatures)
ringModel_event_infos.append(ringModel_event_info)
medianR_event_info = self.medianR_extraction(
ringM_muonFeatures, photon_clusters, event_id)
medianR_event_infos.append(medianR_event_info)
knownC_event_info = self.known_center(event_id,
photon_clusters)
knownC_event_infos.append(knownC_event_info)
data_to_be_saved = [
ringModel_event_infos, medianR_event_infos, knownC_event_infos,
hough_event_infos
]
methods = ["ringM", "medianR", "knownC", "hough"]
for method, data in zip(methods, data_to_be_saved):
self.save_to_file(data, method)
def run_job(inpath, outpath, method=False):
results = []
run = ps.EventListReader(inpath)
number_muons = 0
for event in run:
clusters = ps.PhotonStreamCluster(event.photon_stream)
random_state = np.random.get_state()
np.random.seed(event.photon_stream.number_photons)
if not callable(method):
muon_props = extraction.detection(event, clusters)
else:
muon_props = method(event, clusters)
np.random.set_state(random_state)
if muon_props["is_muon"]:
std_photons_on_ring = muon_ring_std_event(clusters, muon_props)
results.append(std_photons_on_ring)
number_muons += 1
outdir = os.path.dirname(outpath)
os.makedirs(outdir, exist_ok=True)
with open(outpath + ".temp", "wt") as fout:
out = {
"average_fuzz": float(np.average(results)),
"std_fuzz": float(np.std(results)),
"number_muons": number_muons,
}
fout.write(json.dumps(out))
os.rename(outpath + ".temp", outpath)
def gen_features_norm(data_file, lower, upper, sim_file=None):
"""
This generates a certain set of features from photon-stream simulation
or data files that can be used for further analyses.
Inputs:
data_file: location of input data file as string
lower: lower limit for time slice cleaning
upper: upper limit for time slice cleaning
sim_file: location of input simulations file as string
default: corresponding to name of data file
return:
pandas data frame with features
"""
# read in files
if is_simulation_file(data_file):
reader = ps.SimulationReader(
photon_stream_path=data_file,
mmcs_corsika_path=sim_file)
else:
reader = ps.EventListReader(data_file)
# initialisation of list of dicts containing generated data
events = list()
border_pix = get_border_pixel_mask()
x, y = get_pixel_coords()
# loop for events
for event in reader:
lol = event.photon_stream.list_of_lists
image = phs2image(lol, lower, upper)
mask = facttools_cleaning(image, lol, lower, upper)
# empty dict for values
ev = {}
# number of photons in biggest cluster
ev['size'] = image[mask].sum()
if ev['size'] > 0:
border_ph = [(border_pix[i] and mask[i]) for i in range(1440)]
ev['leakage'] = image[border_ph].sum()/ev['size']
ev.update(safe_observation_info(event))
ev.update(calc_hillas_features_image(image, mask))
# append values from dict to list of dicts (events)
events.append(ev)
# save list of dicts in pandas data frame
df = pd.DataFrame(events)
return df
def process(self):
used_paths = split_data(self.raw_paths)[self.split]
for raw_path in used_paths:
if not self.include_proton and "proton" in raw_path:
continue
# load the pickled file from the disk
if osp.exists(osp.join(self.processed_dir, self.split, f"{raw_path}.pt")):
self.processed_filenames.append(f"{raw_path}.pt")
else:
if self.simulated:
mc_truth = raw_path.split(".phs")[0] + ".ch.gz"
event_reader = ps.SimulationReader(
photon_stream_path=raw_path, mmcs_corsika_path=mc_truth
)
else:
event_reader = ps.EventListReader(raw_path)
for event in event_reader:
print(raw_path)
# Convert List of List to Point Cloud, then truncation is simply cutting in the z direction
point_cloud = np.asarray(event.photon_stream.point_cloud)
# Read data from `raw_path`.
data = Data(
pos=point_cloud
) # Just need x,y,z ignore derived features
if self.simulated:
if "proton" in raw_path:
data.event_type = torch.tensor(0, dtype=torch.int8)
elif "gamma" in raw_path:
data.event_type = torch.tensor(1, dtype=torch.int8)
else:
print("No Event Type")
continue
data.energy = torch.tensor(
event.simulation_truth.air_shower.energy, dtype=torch.float
)
data.phi = torch.tensor(
event.simulation_truth.air_shower.phi, dtype=torch.float
)
data.theta = torch.tensor(
event.simulation_truth.air_shower.theta, dtype=torch.float
)
if self.pre_filter is not None and not self.pre_filter(data):
continue
if self.pre_transform is not None:
data = self.pre_transform(data)
torch.save(
data,
osp.join(
self.processed_dir, self.split, "{}.pt".format(raw_path)
),
)
self.processed_filenames.append("{}.pt".format(raw_path))
def main():
gammas = list(filter(lambda a: a.endswith('phs.jsonl.gz'), os.listdir(gamma_path)))
protons = list(filter(lambda a: a.endswith('phs.jsonl.gz'), os.listdir(proton_path)))
data = ps.EventListReader('/net/big-tank/POOL/projects/fact/photon-stream/stream_data/crab/20131101_160.phs.jsonl.gz')
print('Reading data...')
lol_d = []
events = 0
for event in tqdm(data, total=2000):
events += 1
plot_time(lol_d, event)
if events > 2000:
break
print('Reading gamma...')
lol_g = []
events = 0
for file in gammas:
gamma = ps.EventListReader(gamma_path + file)
for event in tqdm(gamma, total=2000):
events += 1
plot_time(lol_g, event)
if events > 2000:
break
if events > 2000:
break
print('Reading proton...')
lol_p = []
events = 0
for file in protons:
proton = ps.EventListReader(proton_path + file)
for event in tqdm(proton, total=2000):
events += 1
plot_time(lol_p, event)
if events > 2000:
break
if events > 2000:
break
safe_plot(lol_d, lol_g, lol_p)
def extract_fuzzParam_from_single_run(self, inpath):
response_resultsR = []
response_resultsH = []
fuzz_resultsR = []
fuzz_resultsH = []
number_muonsH = 0
number_muonsR = 0
mu_event_ids = []
reconstructed_muon_eventsH = []
reconstructed_muon_eventsR = []
run = ps.EventListReader(inpath)
for event_id, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_propsR = ringM(event, photon_clusters)
muon_propsH = hough(event, photon_clusters)
if muon_propsH['is_muon']:
houghResults = self.extraction(muon_propsH, photon_clusters,
event_id)
reconstructed_muon_eventsH.append(houghResults[2])
response_resultsH.append(houghResults[0])
fuzz_resultsH.append(houghResults[1])
number_muonsH += 1
if muon_propsR['is_muon']:
ringM_results = self.extraction(muon_propsR, photon_clusters,
event_id)
reconstructed_muon_eventsR.append(ringM_results[2])
response_resultsR.append(ringM_results[0])
fuzz_resultsR.append(ringM_results[1])
number_muonsR += 1
psf = self.get_point_spread_function(inpath)
responseR_avg, responseR_stdev = self.calculate_average_and_stdev(
response_resultsR)
fuzzR_avg, fuzzR_stdev = self.calculate_average_and_stdev(
fuzz_resultsR)
responseH_avg, responseH_stdev = self.calculate_average_and_stdev(
response_resultsH)
fuzzH_avg, fuzzH_stdev = self.calculate_average_and_stdev(
fuzz_resultsH)
runInfo = {
"reconstructed_muon_eventsH": reconstructed_muon_eventsH,
"responseR_avg": responseR_avg,
"responseR_stdev": responseR_stdev,
"fuzzR_avg": fuzzR_avg,
"fuzzR_stdev": fuzzR_stdev,
"number_muonsH": number_muonsH,
"reconstructed_muon_eventsR": reconstructed_muon_eventsR,
"responseH_avg": responseH_avg,
"responseH_stdev": responseH_stdev,
"fuzzH_avg": fuzzH_avg,
"fuzzH_stdev": fuzzH_stdev,
"number_muonsR": number_muonsR,
"point_spread_function": psf,
"inpath": inpath
}
return runInfo
def cluster_single_run(dir_name, clustering=ps.PhotonStreamCluster):
nsb_run_photons = []
pure_run_photons = []
all_run_photons = []
pure_run_path = os.path.join(dir_name, "pure", "psf_0.sim.phs")
nsb_run_path = os.path.join(dir_name, "NSB", "psf_0.sim.phs")
nsb_run = ps.EventListReader(nsb_run_path)
pure_run = ps.EventListReader(pure_run_path)
for event in nsb_run:
photon_clusters = clustering(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
nsb_cherenkov_photon_stream = photon_clusters.point_cloud
nsb_cherenkov_ps = nsb_cherenkov_photon_stream[cherenkov_cluster_mask]
nsb_run_photons.append(nsb_cherenkov_ps[:, 0:3])
all_photons = event.photon_stream.point_cloud
all_run_photons.append(all_photons)
for muon in pure_run:
pure_photon_stream = muon.photon_stream.point_cloud
pure_run_photons.append(pure_photon_stream)
return (all_run_photons, pure_run_photons, nsb_run_photons)
def do_clustering(self, pure_cherenkov_events_path, events_with_nsb_path):
pure_cherenkov_run = ps.EventListReader(pure_cherenkov_events_path)
nsb_run = ps.EventListReader(events_with_nsb_path)
nsb_run_found_photons = []
pure_cherenkov_run_photons = []
cut_cherenkov = []
all_photons_run = []
for event in nsb_run:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
cherenkov_cluster_mask = photon_clusters.labels >= 0
nsb_cherenkov_photon_stream = photon_clusters.point_cloud
nsb_cherenkov_ps = nsb_cherenkov_photon_stream[
cherenkov_cluster_mask]
cherenkov_photons = self.cut_hist(nsb_cherenkov_ps)
cut_cherenkov.append(cherenkov_photons)
nsb_run_found_photons.append(nsb_cherenkov_ps[:, 0:3])
all_photons = event.photon_stream.point_cloud
all_photons_run.append(all_photons)
for muon in pure_cherenkov_run:
pure_photon_stream = muon.photon_stream.point_cloud
pure_cherenkov_run_photons.append(pure_photon_stream)
return (all_photons_run, pure_cherenkov_run_photons,
nsb_run_found_photons, cut_cherenkov)
def gen_features(data_file, eps=0.1, sim_file=None):
"""
This generates a certain set of features from photon-stream simulation
or data files that can be used for further analyses.
Inputs:
-----------------------------------------
data_file: location of input data file as string
sim_file: location of input simulations file as string
default: corresponding to name of data file
Returns:
-----------------------------------------
pandas data frame with features
"""
# if eps is None:
# eps = 0.1
# read in files
if is_simulation_file(data_file):
reader = ps.SimulationReader(
photon_stream_path=data_file,
mmcs_corsika_path=sim_file)
else:
reader = ps.EventListReader(data_file)
# initialisation of list of dicts containing generated data
events = list()
# loop for events
for event in reader:
# clustering of events
clustering = ps.photon_cluster.PhotonStreamCluster(event.photon_stream, eps=eps)
if clustering.number >= 1:
# empty dict for values
ev = {}
# safe hillas features
ev.update(calc_hillas_features_phs(event.photon_stream, clustering))
# safe observation info
ev.update(safe_observation_info(event))
# append values from dict to list of dicts (events)
events.append(ev)
# save list of dicts in pandas data frame
df = pd.DataFrame(events)
return df
def run_detection(self, inpath):
run = ps.EventListReader(inpath)
path = os.path.normpath(inpath)
split_path = path.split(os.sep)
oa_name = split_path[-2]
oa = re.split('_', oa_name)[2]
preferences = self.read_preferencesFile()
number_of_muons = preferences['--number_of_muons']
found_muons = 0
for event in run:
clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_props = detection(event, clusters)
if muon_props["is_muon"]:
found_muons += 1
return oa, found_muons, number_of_muons
def convert_file(path):
# print('Analyzing {}'.format(photon_stream_file))
try:
reader = ps.SimulationReader(
photon_stream_path=path,
)
except Exception as e:
print(e)
reader = ps.EventListReader(path)
try:
rows = [image_from_event(event) for event in reader]
return Table(rows).as_array()
except ValueError as e:
print('Failed to read data from file: {}'.format(path))
print('PhotonStreamError: {}'.format(e))
def test_muon_detection():
np.random.seed(seed=1)
muon_truth_path = pkg_resources.resource_filename(
'muons',
os.path.join('tests', 'resources', 'muon_sample_20140101_104.csv'))
muon_truth = np.genfromtxt(muon_truth_path)
muon_sample_path = pkg_resources.resource_filename(
'muons',
os.path.join('tests', 'resources',
'20140101_104_muon_sample.phs.jsonl.gz'))
run = ps.EventListReader(muon_sample_path)
true_positives = 0
true_negatives = 0
false_positives = 0
false_negatives = 0
for event in run:
clusters = ps.PhotonStreamCluster(event.photon_stream)
ret = muons.detection(event, clusters)
if ret['is_muon']:
if event.observation_info.event in muon_truth:
true_positives += 1
else:
false_positives += 1
else:
if event.observation_info.event in muon_truth:
false_negatives += 1
else:
true_negatives += 1
precision = true_positives / (true_positives + false_positives)
sensitivity = true_positives / (true_positives + false_negatives)
print('precision', precision)
print('sensitivity', sensitivity)
assert precision > 0.995
assert sensitivity > 0.76
def test_extraction():
Analysis = pfa.PSF_FuzzAnalysis(
preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir)
simulationFile = os.path.join(
fileDir, "resources", "100simulations_psf0.0.sim.phs")
run = ps.EventListReader(simulationFile)
event_id = 3
for i, event in enumerate(run):
if i == event_id:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_props = {
"muon_ring_cx": 0,
"muon_ring_cy": 0,
"muon_ring_r": 1,
"mean_arrival_time_muon_cluster": 2,
"muon_ring_overlapp_with_field_of_view": 3,
"number_of_photons": 3
}
returns = Analysis.extraction(
muon_props, photon_clusters, event_id)
assert len(returns) == 3
def test_get_fuzziness_parameters():
Analysis = pfa.PSF_FuzzAnalysis(
preferencesFile, maximum_PSF, steps, scoop_hosts, output_dir)
simulationFile = os.path.join(
fileDir, "resources", "100simulations_psf0.0.sim.phs")
run = ps.EventListReader(simulationFile)
for i, event in enumerate(run):
if i == 47:
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_props = {
"muon_ring_cx": 0,
"muon_ring_cy": 0,
"muon_ring_r": 1,
"mean_arrival_time_muon_cluster": 2,
"muon_ring_overlapp_with_field_of_view": 3,
"number_of_photons": 3
}
normed_response, fuzziness_stdParam = Analysis.get_fuzziness_parameters(
photon_clusters, muon_props)
assert (
isinstance(normed_response, Number) and
isinstance(fuzziness_stdParam, Number)
)
def count_events(self):
if self.num_events < 0:
count = 0
for index, file in enumerate(self.paths):
try:
print("Trying...")
crab_reader = ps.EventListReader(file)
for event in crab_reader:
df_event = self.dl2_file.loc[
(self.dl2_file["event_num"] == event.observation_info.event)
& (self.dl2_file["night"] == event.observation_info.night)
& (self.dl2_file["run_id"] == event.observation_info.run)
]
if not df_event.empty:
count += 1
print(count)
except Exception as e:
print(str(e))
print(count)
print("\n")
self.num_events = count
return count
return self.num_events
def from_observation(phs_path):
event_list = ps.EventListReader(phs_path)
features = []
for event in event_list:
if event.observation_info.trigger_type == PHYSICS_TRIGGER:
cluster = ps.PhotonStreamCluster(event.photon_stream)
cluster = reject.early_or_late_clusters(cluster)
f = raw_features(photon_stream=event.photon_stream,
cluster=cluster)
f['type'] = ps.io.binary.OBSERVATION_EVENT_TYPE_KEY
f['az'] = np.deg2rad(event.az)
f['zd'] = np.deg2rad(event.zd)
f['night'] = event.observation_info.night
f['run'] = event.observation_info.run
f['event'] = event.observation_info.event
f['time'] = event.observation_info._time_unix_s + event.observation_info._time_unix_us / 1e6
features.append(f)
triggered = pd.DataFrame(features)
triggered = tools.reduce_DataFrame_to_32_bit(triggered)
return triggered
def main(method, path, file, feat, number):
border_pix = get_border_pixel_mask()
print("Reading in facttools dl1 file...")
t = Table.read(
'/net/big-tank/POOL/projects/fact/photon-stream/facttools/crab/{}_dl1.fits'
.format(file))
# print("Reading in facttools dl2 file...")
# dl2 = read_data('/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/crab.hdf5', key='events')
print("Reading in PhotonStream data file...")
reader = ps.EventListReader(
'/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'
.format(path, file))
print("Reading in PhotonStream dl2 data...")
ph_dl2 = read_data('/home/ksedlaczek/OUT_0.10/crab_data_precuts.hdf5',
key='events')
print("Done...")
event = next(reader)
d_delta = []
disp_prediction = []
for i in tqdm(range(number)):
if is_simulation_event(event):
event_num_phs = event.simulation_truth.event
reuse_phs = event.simulation_truth.reuse
run_id_phs = event.simulation_truth.run
else:
event_num_phs = event.observation_info.event
reuse_phs = 42
run_id_phs = event.observation_info.run
if path != 'crab':
run_id = file
event_num = t[i]['MCorsikaEvtHeader.fEvtNumber']
reuse = t[i]['MCorsikaEvtHeader.fNumReuse']
else:
run_id = file
event_num = t[i]['EventNum']
reuse = 42
assert run_id != run_id_phs
while (event_num_phs != event_num or reuse != reuse_phs):
event = next(reader)
if is_simulation_event(event):
event_num_phs = event.simulation_truth.event
reuse_phs = event.simulation_truth.reuse
run_id_phs = event.simulation_truth.run
else:
event_num_phs = event.observation_info.event
reuse_phs = 42
run_id_phs = event.observation_info.run
lol = event.photon_stream.list_of_lists
# cut away unwanted time slices
# lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol]
image = phs2image(lol)
disp_prediction.append(
ph_dl2.query('night == 20131104 & run == 162 & event == {}'.format(
event_num))['disp_prediction'].values)
cleaned_pix = t[i]['shower']
d_delta.append(calc_delta_delta(event, cleaned_pix))
mask = np.array([
True if disp_prediction[i] < 0 else False
for i in range(len(disp_prediction))
])
plt.figure()
plt.hist(np.array(d_delta)[mask],
bins=np.linspace(-np.pi / 2, 3 / 2 * np.pi, 101),
alpha=0.4,
label='disp < 0')
plt.hist(np.array(d_delta)[~mask],
bins=np.linspace(-np.pi / 2, 3 / 2 * np.pi, 101),
alpha=0.4,
label='disp >= 0')
# plt.hist(np.array(d_delta)[mask], bins=100, alpha=0.4, label='disp < 0')
# plt.hist(np.array(d_delta)[~mask], bins=100, alpha=0.4, label='disp >= 0')
plt.title(
r'$\mathrm{\Delta}\delta$ between phs on facttools cleaning to $\delta_{\mathrm{true}}$ per image'
)
plt.legend(loc='best')
plt.xlabel(r'$\mathrm{\Delta}\delta$ / rad')
plt.savefig('delta_true_diff_hist_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
def run_job(self, job):
results = []
inpath = job["inpath"]
output_path_responseH = job["output_path_responseH"]
output_path_stdevR = job["output_path_stdevR"]
output_path_responseR = job["output_path_responseR"]
output_path_stdevH = job["output_path_stdevH"]
muonCountH = 0
muonCountR = 0
run = ps.EventListReader(inpath)
muon_ring_featuresR = []
muon_ring_featuresH = []
fuzziness_stdParamRs = []
fuzziness_stdParamHs = []
normed_responseRs = []
normed_responseHs = []
fuzz_paramsH = []
for event_id, event in enumerate(run):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_propsR = ringM_detection(event, photon_clusters)
muon_propsH = detection(event, photon_clusters)
if muon_propsH["is_muon"]:
muonCountH += 1
normed_responseH, fuzziness_stdParamH = (
self.get_fuzziness_parameters(photon_clusters,
muon_propsH))
muon_ring_featureH = [
muon_propsH["muon_ring_cx"], muon_propsH["muon_ring_cy"],
muon_propsH["muon_ring_r"]
]
muon_ring_featuresH.append(muon_ring_featureH)
fuzziness_stdParamHs.append(fuzziness_stdParamH)
normed_responseHs.append(normed_responseH)
if muon_propsR["is_muon"]:
muonCountR += 1
normed_responseR, fuzziness_stdParamR = (
self.get_fuzziness_parameters(photon_clusters,
muon_propsR))
muon_ring_featureR = [
muon_propsR["muon_ring_cx"], muon_propsR["muon_ring_cy"],
muon_propsR["muon_ring_r"]
]
fuzziness_stdParamRs.append(fuzziness_stdParamR)
normed_responseRs.append(normed_responseR)
muon_ring_featuresR.append(muon_ring_featureR)
fact_path = fact.path.parse(inpath)
night = fact_path["night"]
run = fact_path["run"]
filename = str(night) + "_" + str(run) + ".csv"
output_dirR = os.path.dirname(output_path_stdevR)
output_dirH = os.path.dirname(output_path_stdevH)
if not os.path.isdir(output_dirR):
os.makedirs(output_dirR, exist_ok=True)
if not os.path.isdir(output_dirH):
os.makedirs(output_dirH, exist_ok=True)
self.save_to_file("ringM", output_dirR, filename, muon_ring_featuresR)
self.save_to_file("hough", output_dirH, filename, muon_ring_featuresH)
self.save_fuzz_param(output_path_stdevR, fuzziness_stdParamRs,
muonCountR)
self.save_fuzz_param(output_path_responseR, normed_responseRs,
muonCountR)
self.save_fuzz_param(output_path_stdevH, fuzziness_stdParamHs,
muonCountH)
self.save_fuzz_param(output_path_responseH, normed_responseHs,
muonCountH)
return 0
def main(method, path, file, feat, number):
border_pix = get_border_pixel_mask()
print("Reading in facttools dl1 file...")
t = Table.read('/net/big-tank/POOL/projects/fact/photon-stream/facttools/crab/{}_dl1.fits'.format(file))
print("Reading in facttools dl2 file...")
dl2 = read_data('/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/crab.hdf5', key='events')
print("Reading in PhotonStream data file...")
reader = ps.EventListReader('/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'.format(path, file))
print("Done...")
event = next(reader)
fig, axs = plt.subplots(3, 1, figsize=(4, 10), constrained_layout=True)
plots = [camera(np.zeros(1440), cmap='inferno', ax=ax) for ax in axs]
cbars = [fig.colorbar(plot, ax=ax) for plot, ax in zip(plots, axs)]
plots[1].set_cmap('RdBu_r')
with PdfPages('pe_difference_{}_{}.pdf'.format(feat, file)) as pdf:
for i in tqdm(range(number)):
if is_simulation_event(event):
event_num_phs = event.simulation_truth.event
reuse_phs = event.simulation_truth.reuse
run_id_phs = event.simulation_truth.run
else:
event_num_phs = event.observation_info.event
reuse_phs = 42
run_id_phs = event.observation_info.run
if path != 'crab':
run_id = file
event_num = t[i]['MCorsikaEvtHeader.fEvtNumber']
reuse = t[i]['MCorsikaEvtHeader.fNumReuse']
else:
run_id = file
event_num = t[i]['EventNum']
reuse = 42
assert run_id != run_id_phs
while (event_num_phs != event_num or reuse != reuse_phs):
event = next(reader)
if is_simulation_event(event):
event_num_phs = event.simulation_truth.event
reuse_phs = event.simulation_truth.reuse
run_id_phs = event.simulation_truth.run
else:
event_num_phs = event.observation_info.event
reuse_phs = 42
run_id_phs = event.observation_info.run
lol = event.photon_stream.list_of_lists
# cut away unwanted time slices
# lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol]
image = phs2image(lol)
if method == 'DBSCAN':
clustering = ps.photon_cluster.PhotonStreamCluster(event.photon_stream)
biggest_cluster = np.argmax(np.bincount(clustering.labels[clustering.labels != -1]))
mask = clustering.labels == biggest_cluster
cleaned_pix_phs = np.zeros(len(image), dtype=bool)
k = 0
cleaned_img = np.zeros(len(image))
for h in range(len(lol)):
for j in range(len(lol[h])):
k += 1
if mask[k-1]:
cleaned_pix_phs[h] = True
else:
cleaned_pix_phs = facttools_cleaning(image, lol, picture_thresh, boundary_thresh)
if sum(cleaned_pix_phs) < 1:
break
cleaned_pix = t[i]['shower']
t[i]['photoncharge'][t[i]['photoncharge'] < 0] = 0.0
pe_difference = image - t[i]['photoncharge']
max_abs = np.max(np.abs(pe_difference))
plots[0].set_array(image)
plots[1].set_array(pe_difference)
plots[2].set_array(t[i]['photoncharge'])
plots[0].autoscale()
plots[1].set_clim(-max_abs, max_abs)
plots[2].autoscale()
# mark_pixel(t[i]['shower'], color=(128/255, 186/255, 38/255), linewidth=2.5)
for ax in axs:
ax.axis('off')
for cbar, plot in zip(cbars, plots):
cbar.update_bruteforce(plot)
# embed()
if is_simulation_event(event):
fig.suptitle('run {} event {} reuse {} mean {:.2f}'.format(run_id, event_num, reuse, np.mean(pe_difference)))
else:
# fig.suptitle('{} event {} mean {:.2f}'.format(file, event_num, np.mean(pe_difference)))
fig.suptitle('{} event {}'.format(file[:8] + ' ' + file[9:], event_num))
pdf.savefig(fig)
def main(method, path, file, feat, number):
border_pix = get_border_pixel_mask()
print("Reading in facttools dl1 file...")
t = Table.read('/net/big-tank/POOL/projects/fact/photon-stream/facttools/{}/{}_dl1.fits'.format(path, file))
print("Reading in facttools dl2 file...")
dl2 = read_data('/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/{}.hdf5'.format(path), key='events')
print("Reading in PhotonStream data file...")
reader = ps.EventListReader('/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'.format(path, file))
print("Done...")
event = next(reader)
all_pe_diff_mean = []
all_pe_diff = []
delta_delta = []
delta_delta_diff = []
delta_delta_diff_perc = []
d_delta = []
for i in tqdm(range(number)):
if is_simulation_event(event):
event_num_phs = event.simulation_truth.event
reuse_phs = event.simulation_truth.reuse
run_id_phs = event.simulation_truth.run
else:
event_num_phs = event.observation_info.event
reuse_phs = 42
run_id_phs = event.observation_info.run
if path != 'crab':
run_id = file
event_num = t[i]['MCorsikaEvtHeader.fEvtNumber']
reuse = t[i]['MCorsikaEvtHeader.fNumReuse']
else:
run_id = file
event_num = t[i]['EventNum']
reuse = 42
assert run_id != run_id_phs
while (event_num_phs != event_num or reuse != reuse_phs):
event = next(reader)
if is_simulation_event(event):
event_num_phs = event.simulation_truth.event
reuse_phs = event.simulation_truth.reuse
run_id_phs = event.simulation_truth.run
else:
event_num_phs = event.observation_info.event
reuse_phs = 42
run_id_phs = event.observation_info.run
lol = event.photon_stream.list_of_lists
# cut away unwanted time slices
# lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol]
image = phs2image(lol)
if method == 'DBSCAN':
clustering = ps.photon_cluster.PhotonStreamCluster(event.photon_stream)
if clustering.number < 1:
continue
biggest_cluster = np.argmax(np.bincount(clustering.labels[clustering.labels != -1]))
mask = clustering.labels == biggest_cluster
cleaned_pix_phs = np.zeros(len(image), dtype=bool)
k = 0
cleaned_img = np.zeros(len(image))
for h in range(len(lol)):
for j in range(len(lol[h])):
k += 1
if mask[k-1]:
cleaned_pix_phs[h] = True
else:
cleaned_pix_phs = facttools_cleaning(image, lol, picture_thresh, boundary_thresh)
if sum(cleaned_pix_phs) < 1:
continue
cleaned_pix = t[i]['shower']
t[i]['photoncharge'][t[i]['photoncharge'] < 0] = 0.0
pe_difference = image - t[i]['photoncharge']
if is_simulation_event(event):
delta = np.rad2deg(dl2.query('night == 20131104 & run_id == 162 & event_num == {}'.format(t[i]['EventNum']))['delta'].values[0])
else:
delta = np.rad2deg(dl2.query('night == 20131104 & run_id == 162 & event_num == {}'.format(t[i]['EventNum']))['delta'].values[0])
delta_diff_same = calc_delta(image, cleaned_pix) - delta
delta_diff_whole = calc_delta(image, cleaned_pix_phs) - delta
delta_diff_whole_perc = calc_delta_perc(image, cleaned_pix_phs) - delta
for val in pe_difference:
all_pe_diff.append(val)
all_pe_diff_mean.append(np.mean(pe_difference))
delta_delta.append(delta_diff_same)
delta_delta_diff.append(delta_diff_whole)
delta_delta_diff_perc.append(delta_diff_whole_perc)
d_delta.append(calc_delta_delta(event, cleaned_pix))
plt.figure()
plt.hist(all_pe_diff_mean, bins=100, histtype='step')
plt.title('Means of pe differences per image')
plt.tight_layout()
plt.savefig('means_hist_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(all_pe_diff, bins=100, histtype='step', density=True)
plt.title('PE differences per pixel')
plt.tight_layout()
plt.savefig('diffs_hist_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(all_pe_diff, bins=100, histtype='step')
plt.title('PE differences per pixel')
plt.semilogy()
plt.ylabel('events')
plt.xlabel(r'$\symup{\Delta}$PE')
plt.tight_layout()
plt.savefig('diffs_hist_{}_{}_{}_logy.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(delta_delta, bins=70, histtype='step', density=True)
plt.title('$\mathrm{\Delta}\delta$ between phs and facttools')
plt.tight_layout()
plt.savefig('delta_hist_same_pixels_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(delta_delta_diff, bins=70, histtype='step', density=True)
plt.title(r'$\mathup{\Delta}\delta$ between phs and facttools')
plt.xlabel(r'$\mathrm{\Delta}\delta / \textdegree$')
plt.tight_layout()
plt.savefig('delta_diff_hist_different_cleanings_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(delta_delta_diff_perc, bins=70, histtype='step', density=True)
plt.title(r'$\mathup{\Delta}\delta$ between phs and facttools')
plt.xlabel(r'$\mathrm{\Delta}\delta / \textdegree$')
plt.tight_layout()
plt.savefig('delta_diff_hist_perc_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(delta_delta_diff, bins=70, histtype='step', density=True, label='DBSCAN')
plt.hist(delta_delta_diff_perc, bins=70, histtype='step', density=True, label='DBSCAN + > 1%')
plt.title(r'$\mathup{\Delta}\delta$ between phs and facttools')
plt.xlabel(r'$\mathrm{\Delta}\delta / \textdegree$')
plt.legend()
plt.tight_layout()
plt.savefig('delta_diff_hist_perc_and_normal_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
plt.figure()
plt.hist(d_delta, bins=100, histtype='step', density=True)
plt.title(r'$\mathrm{\Delta}\delta$ between phs on facttools cleaning to $\delta_{\mathrm{true}}$ per image')
plt.tight_layout()
plt.savefig('delta_true_diff_hist_{}_{}_{}.pdf'.format(method, feat, file))
plt.clf()
def event_processor(
self, directory, clean_images=False, only_core=True, clump_size=20
):
for index, file in enumerate(self.paths):
file_name = file.split("/")[-1].split(".phs")[0]
try:
sim_reader = ps.EventListReader(file)
counter = 0
for event in sim_reader:
df_event = self.dl2_file.loc[
(self.dl2_file["event_num"] == event.observation_info.event)
& (self.dl2_file["night"] == event.observation_info.night)
& (self.dl2_file["run_id"] == event.observation_info.run)
]
if not df_event.empty:
counter += 1
if os.path.isfile(
os.path.join(
directory,
"clump" + str(clump_size),
str(file_name) + "_" + str(counter),
)
) and os.path.isfile(
os.path.join(
directory,
"core" + str(clump_size),
str(file_name) + "_" + str(counter),
)
):
print("True: " + str(file_name) + "_" + str(counter))
continue
if clean_images:
# Do it for no clumps, all clump, and only core into different subfolders
all_photons, clump_photons, core_photons = self.clean_image(
event, only_core=only_core, min_samples=clump_size
)
if core_photons is None:
print("No Clumps, skip")
continue
else:
for key, photon_set in {
"no_clean": all_photons,
"clump": clump_photons,
"core": core_photons,
}.items():
event.photon_stream.raw = photon_set
# In the event chosen from the file
# Each event is the same as each line below
source_pos_x = df_event["source_position_x"].values[
0
]
source_pos_y = df_event["source_position_y"].values[
0
]
timestamp = (
df_event["timestamp"]
.values[0]
.astype(datetime.datetime)
)
event_photons = event.photon_stream.list_of_lists
zd_deg = event.zd
az_deg = event.az
cog_x = df_event["cog_x"].values[0]
cog_y = df_event["cog_y"].values[0]
sky_source_az = df_event[
"source_position_az"
].values[0]
sky_source_zd = df_event[
"source_position_zd"
].values[0]
zd_deg1 = df_event["pointing_position_zd"].values[0]
az_deg1 = df_event["pointing_position_az"].values[0]
event_num = event.observation_info.event
night = event.observation_info.night
run = event.observation_info.run
data_dict = [
[
event_photons,
timestamp,
zd_deg,
az_deg,
cog_x,
cog_y,
sky_source_az,
sky_source_zd,
zd_deg1,
az_deg1,
source_pos_x,
source_pos_y,
event_num,
night,
run,
],
{
"Image": 0,
"Timestamp": 1,
"Zd_Deg": 2,
"Az_Deg": 3,
"COG_X": 4,
"COG_Y": 5,
"Source_Position_Az": 6,
"Source_Position_Zd": 7,
"Pointing_Position_Zd": 8,
"Pointing_Position_Az": 9,
"Source_Position_X": 10,
"Source_Position_Y": 11,
"Event_Number": 12,
"Night": 13,
"Run": 14,
},
]
if key != "no_clean":
with open(
os.path.join(
directory,
key + str(clump_size),
str(file_name) + "_" + str(counter),
),
"wb",
) as event_file:
pickle.dump(data_dict, event_file)
else:
if not os.path.isfile(
os.path.join(
directory,
key,
str(file_name) + "_" + str(counter),
)
):
with open(
os.path.join(
directory,
key,
str(file_name) + "_" + str(counter),
),
"wb",
) as event_file:
pickle.dump(data_dict, event_file)
else:
# In the event chosen from the file
# Each event is the same as each line below
# Each event is the same as each line below
source_pos_x = df_event["source_position_x"].values[0]
source_pos_y = df_event["source_position_y"].values[0]
timestamp = (
df_event["timestamp"]
.values[0]
.astype(datetime.datetime)
)
event_photons = event.photon_stream.list_of_lists
zd_deg = event.zd
az_deg = event.az
cog_x = df_event["cog_x"].values[0]
cog_y = df_event["cog_y"].values[0]
sky_source_az = df_event["source_position_az"].values[0]
sky_source_zd = df_event["source_position_zd"].values[0]
zd_deg1 = df_event["pointing_position_zd"].values[0]
az_deg1 = df_event["pointing_position_az"].values[0]
event_num = event.observation_info.event
night = event.observation_info.night
run = event.observation_info.run
data_dict = [
[
event_photons,
timestamp,
zd_deg,
az_deg,
cog_x,
cog_y,
sky_source_az,
sky_source_zd,
zd_deg1,
az_deg1,
source_pos_x,
source_pos_y,
event_num,
night,
run,
],
{
"Image": 0,
"Timestamp": 1,
"Zd_Deg": 2,
"Az_Deg": 3,
"COG_X": 4,
"COG_Y": 5,
"Source_Position_Az": 6,
"Source_Position_Zd": 7,
"Pointing_Position_Zd": 8,
"Pointing_Position_Az": 9,
"Source_Position_X": 10,
"Source_Position_Y": 11,
"Event_Number": 12,
"Night": 13,
"Run": 14,
},
]
with open(
os.path.join(
directory, str(file_name) + "_" + str(counter)
),
"wb",
) as event_file:
pickle.dump(data_dict, event_file)
except Exception as e:
print(str(e))
pass
def single_processor(
self,
normalize=False,
collapse_time=False,
final_slices=5,
as_channels=False,
clean_images=False,
):
while True:
print("New Crab")
for index, file in enumerate(self.paths):
try:
sim_reader = ps.EventListReader(file)
for event in sim_reader:
data = []
df_event = self.dl2_file.loc[
(self.dl2_file["event_num"] == event.observation_info.event)
& (self.dl2_file["night"] == event.observation_info.night)
& (self.dl2_file["run_id"] == event.observation_info.run)
]
if not df_event.empty:
if clean_images:
event = self.clean_image(event)
# In the event chosen from the file
# Each event is the same as each line below
source_pos_x = df_event["source_position_x"].values[0]
source_pos_y = df_event["source_position_y"].values[0]
energy = (
df_event["timestamp"]
.values[0]
.astype(datetime.datetime)
)
event_photons = event.photon_stream.list_of_lists
zd_deg = event.zd
az_deg = event.az
cog_x = df_event["cog_x"].values[0]
cog_y = df_event["cog_y"].values[0]
sky_source_az = df_event["source_position_az"].values[0]
sky_source_zd = df_event["source_position_zd"].values[0]
zd_deg1 = df_event["pointing_position_zd"].values[0]
az_deg1 = df_event["pointing_position_az"].values[0]
event_num = event.observation_info.event
night = event.observation_info.night
run = event.observation_info.run
input_matrix = np.zeros(
[self.shape[1], self.shape[2], self.shape[3]]
)
chid_to_pixel = self.rebinning[0]
pixel_index_to_grid = self.rebinning[1]
for index in range(1440):
for element in chid_to_pixel[index]:
coords = pixel_index_to_grid[element[0]]
for value in event_photons[index]:
if self.end >= value >= self.start:
input_matrix[coords[0]][coords[1]][
value - self.start
] += (element[1] * 1)
data.append(
[
np.fliplr(np.rot90(input_matrix, 3)),
energy,
zd_deg,
az_deg,
source_pos_x,
source_pos_y,
sky_source_zd,
sky_source_az,
zd_deg1,
az_deg1,
event_num,
night,
run,
cog_x,
cog_y,
]
)
# need to do the format thing here, and add auxiliary structure
data_format = {
"Image": 0,
"Timestamp": 1,
"Zd_Deg": 2,
"Az_Deg": 3,
"Source_X": 4,
"Source_Y": 5,
"Theta": 6,
}
data = self.format(data)
if normalize:
data = list(data)
data[0] = self.normalize_image(data[0])
data = tuple(data)
if collapse_time:
data = list(data)
data[0] = self.collapse_image_time(
data[0], final_slices, as_channels
)
data = tuple(data)
yield data, data_format
except Exception as e:
print(str(e))
def batch_processor(self, clean_images=False):
self.init()
for index, file in enumerate(self.paths):
print(file)
try:
sim_reader = ps.EventListReader(file)
data = []
for event in sim_reader:
df_event = self.dl2_file.loc[
(self.dl2_file["event_num"] == event.observation_info.event)
& (self.dl2_file["night"] == event.observation_info.night)
& (self.dl2_file["run_id"] == event.observation_info.run)
]
if not df_event.empty:
if clean_images:
event = self.clean_image(event)
# In the event chosen from the file
# Each event is the same as each line below
source_pos_x = df_event["source_position_x"].values[0]
source_pos_y = df_event["source_position_y"].values[0]
energy = (
df_event["timestamp"].values[0].astype(datetime.datetime)
)
event_photons = event.photon_stream.list_of_lists
zd_deg = event.zd
az_deg = event.az
cog_x = df_event["cog_x"].values[0]
cog_y = df_event["cog_y"].values[0]
sky_source_az = df_event["source_position_az"].values[0]
sky_source_zd = df_event["source_position_zd"].values[0]
zd_deg1 = df_event["pointing_position_zd"].values[0]
az_deg1 = df_event["pointing_position_az"].values[0]
event_num = event.observation_info.event
night = event.observation_info.night
run = event.observation_info.run
input_matrix = np.zeros(
[self.shape[1], self.shape[2], self.shape[3]]
)
chid_to_pixel = self.rebinning[0]
pixel_index_to_grid = self.rebinning[1]
for index in range(1440):
for element in chid_to_pixel[index]:
coords = pixel_index_to_grid[element[0]]
for value in event_photons[index]:
if self.end >= value >= self.start:
input_matrix[coords[0]][coords[1]][
value - self.start
] += (element[1] * 1)
data.append(
[
np.fliplr(np.rot90(input_matrix, 3)),
energy,
zd_deg,
az_deg,
source_pos_x,
source_pos_y,
sky_source_zd,
sky_source_az,
zd_deg1,
az_deg1,
event_num,
night,
run,
cog_x,
cog_y,
]
)
yield data
except Exception as e:
print(str(e))
def is_simulation_file(input_file):
reader = ps.EventListReader(input_file)
event = next(reader)
return hasattr(event, 'simulation_truth')
def extract_muons_from_run(
input_run_path,
output_run_path,
output_run_header_path
):
"""
Detects and extracts muon candidate events from a run. The muon candidate
events are exported into a new output run. In addidion a header for the
muon candidates is exported.
Parameter
---------
input_run_path Path to the input run.
output_run_path Path to the output run of muon candidates.
output_run_header_path Path to the binary output run header.
Binary Output Format Run Header
-------------------------------
for each muon candidate:
1) uint32 Night
2) uint32 Run ID
3) uint32 Event ID
4) uint32 unix time seconds [s]
5) uint32 unix time micro seconds modulo full seconds [us]
6) float32 Pointing zenith distance [deg]
7) float32 Pointing azimuth [deg]
8) float32 muon ring center x [deg]
9) float32 muon ring center y [deg]
10) float32 muon ring radius [deg]
11) float32 mean arrival time muon cluster [s]
12) float32 muon ring overlapp with field of view (0.0 to 1.0) [1]
13) float32 number of photons muon cluster [1]
"""
run = ps.EventListReader(input_run_path)
with gzip.open(output_run_path, 'wt') as f_muon_run, \
open(output_run_header_path, 'wb') as f_muon_run_header:
for event in run:
if (
event.observation_info.trigger_type ==
FACT_PHYSICS_SELF_TRIGGER
):
photon_clusters = ps.PhotonStreamCluster(event.photon_stream)
muon_features = detection(event, photon_clusters)
if muon_features['is_muon']:
# EXPORT EVENT in JSON
event_dict = ps.io.jsonl.event_to_dict(event)
json.dump(event_dict, f_muon_run)
f_muon_run.write('\n')
# EXPORT EVENT header
head1 = np.zeros(5, dtype=np.uint32)
head1[0] = event.observation_info.night
head1[1] = event.observation_info.run
head1[2] = event.observation_info.event
head1[3] = event.observation_info._time_unix_s
head1[4] = event.observation_info._time_unix_us
head2 = np.zeros(8, dtype=np.float32)
head2[0] = event.zd
head2[1] = event.az
head2[2] = muon_features['muon_ring_cx']*rad2deg
head2[3] = muon_features['muon_ring_cy']*rad2deg
head2[4] = muon_features['muon_ring_r']*rad2deg
head2[5] = muon_features['mean_arrival_time_muon_cluster']
head2[6] = muon_features[
'muon_ring_overlapp_with_field_of_view'
]
head2[7] = muon_features['number_of_photons']
f_muon_run_header.write(head1.tobytes())
f_muon_run_header.write(head2.tobytes())
def main(method, path, file, feat, number):
border_pix = get_border_pixel_mask()
if method == "thresholds":
reader = ps.EventListReader(
'/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'
.format(path, file))
with PdfPages('cleaning_thresh_{}_{}.pdf'.format(feat, file)) as pdf:
for i in tqdm(range(number)):
fig = plt.figure()
ax = fig.add_axes([0.05, 0.05, 0.9, 0.9])
#ax.set_axis_off()
event = next(reader)
lol = event.photon_stream.list_of_lists
lol = [[t for t in l if ((35 <= t) & (t < 75))] for l in lol]
image = phs2image(lol) #, lower=30, upper=70)
cleaned_pix = facttools_cleaning(image, lol, 35, 75,
picture_thresh,
boundary_thresh)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
arrival_times = np.array([np.nanmedian(l) for l in lol])
# cleaned_pix = cleaning(image, lol, picture_thresh, boundary_thresh)
if len(cleaned_pix[cleaned_pix != 0]) > 1:
# border_ph = [(border_pix[i] and cleaned_pix[i]) for i in range(1440)]
# leakage = image[border_ph].sum()/image[cleaned_pix].sum()
df = calc_hillas_features_image(image, cleaned_pix)
# ell = Ellipse(
# [df['cog_x'], df['cog_y']],
# df['length']*2,
# df['width']*2,
# angle=np.rad2deg(df['delta']),
# fill=False, linewidth=2, color='b'
# )
# ax.add_patch(ell)
ell = Ellipse([df['cog_x'], df['cog_y']],
df['length'] * 4,
df['width'] * 4,
angle=np.rad2deg(df['delta']),
fill=False,
linewidth=1.5,
color='b')
# ax.add_patch(ell)
if is_simulation_event(event):
fig.suptitle('run {} event {} reuse {}'.format(
event.simulation_truth.run,
event.simulation_truth.event,
event.simulation_truth.reuse))
else:
fig.suptitle('{} event {} delta {}'.format(
file, event.observation_info.event, df['delta']))
if feat == 'arrival_times':
with warnings.catch_warnings():
warnings.simplefilter("ignore")
x = arrival_times - np.nanmean(arrival_times)
x[np.isnan(x)] = 0
c = camera(x, cmap='Spectral', ax=ax)
mark_pixel(cleaned_pix, color='k', linewidth=2.5)
else:
c = camera(image, cmap='viridis', ax=ax)
mark_pixel(cleaned_pix,
color=(128 / 255, 186 / 255, 38 / 255),
linewidth=2.5)
ax.axis('off')
fig.colorbar(c)
pdf.savefig(fig)
ax.cla()
plt.close(fig)
if method == "DBSCAN":
reader = ps.EventListReader(
'/net/big-tank/POOL/projects/fact/photon-stream/stream_data/{}/{}.phs.jsonl.gz'
.format(path, file))
with PdfPages('cleaning_DBSCAN_biggest_{}_{}.pdf'.format(feat,
file)) as pdf:
for i in tqdm(range(number)):
fig = plt.figure()
ax = fig.add_axes([0.05, 0.05, 0.9, 0.9])
event = next(reader)
# clustering of events
clustering = ps.photon_cluster.PhotonStreamCluster(
event.photon_stream)
if clustering.number > 0:
lol = event.photon_stream.list_of_lists
image = phs2image(lol)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
arrival_times = np.array(
[np.nanmedian(l) for l in lol])
# biggest cluster:
biggest_cluster = np.argmax(
np.bincount(
clustering.labels[clustering.labels != -1]))
mask = clustering.labels == biggest_cluster
# mask = clustering.labels != -1
xyt = event.photon_stream.point_cloud
x, y, t = xyt.T
cleaned_pix = np.zeros(len(image), dtype=bool)
k = 0
cleaned_img = np.zeros(len(image))
for i in range(len(lol)):
for j in range(len(lol[i])):
k += 1
if mask[k - 1]:
cleaned_pix[i] = True
cleaned_img[i] += 1
cleaned_pix_perc = np.zeros(1440, dtype=bool)
for i in range(1440):
if cleaned_pix[i] and (cleaned_img[i] >
mask.sum() / 200):
cleaned_pix_perc[i] = True
df = calc_hillas_features_phs(event.photon_stream,
clustering)
# ell = Ellipse(
# [df['cog_x'], df['cog_y']],
# df['length']*2,
# df['width']*2,
# angle=np.rad2deg(df['delta']),
# fill=False, linewidth=2, color='b'
# )
# ax.add_patch(ell)
ell = Ellipse([df['cog_x'], df['cog_y']],
df['length'] * 4,
df['width'] * 4,
angle=np.rad2deg(df['delta']),
fill=False,
linewidth=1.5,
color='b')
# ax.add_patch(ell)
if is_simulation_event(event):
fig.suptitle('run {} event {} reuse {}'.format(
event.simulation_truth.run,
event.simulation_truth.event,
event.simulation_truth.reuse))
else:
fig.suptitle('{} event {} delta {:.2f}'.format(
file, event.observation_info.event,
np.rad2deg(df['delta'])))
if feat == 'arrival_times':
with warnings.catch_warnings():
warnings.simplefilter("ignore")
x = arrival_times - np.nanmean(arrival_times)
c = camera(x, cmap='viridis', ax=ax)
mark_pixel(cleaned_pix,
color=(128 / 255, 186 / 255, 38 / 255),
linewidth=2.5)
else:
c = camera(image, cmap='viridis', ax=ax)
mark_pixel(cleaned_pix,
color=(128 / 255, 186 / 255, 38 / 255),
linewidth=2.5)
mark_pixel(cleaned_pix_perc,
color='red',
linewidth=1.5)
ax.axis('off')
fig.colorbar(c)
pdf.savefig(fig)
ax.cla()
plt.close(fig)
if method == "facttools":
print('facttools')
with PdfPages('cleaning_facttools_{}_{}.pdf'.format(feat,
file)) as pdf:
t = Table.read(
'/net/big-tank/POOL/projects/fact/photon-stream/facttools/{}/{}_dl1.fits'
.format(path, file))
dl2 = read_data(
'/home/ksedlaczek/Packages/open_crab_sample_analysis/dl2/crab.hdf5',
key='events')
for i in tqdm(range(number)):
fig = plt.figure()
ax = fig.add_axes([0.05, 0.05, 0.9, 0.9])
# if path != 'crab':
# fig.suptitle('run {} event {} reuse {}'.format(file, t[i]['MCorsikaEvtHeader.fEvtNumber'], t[i]['MCorsikaEvtHeader.fNumReuse']))
# else:
#
# fig.suptitle('{} event {} delta {:.4f}'.format(file, t[i]['EventNum'], dl2.query('night == 20131104 & run_id == 162 & event_num == {}'.format(t[i]['EventNum']))['delta'].values[0]))
t[i]['photoncharge'][t[i]['photoncharge'] < 0] = 0.0
if feat == 'arrival_times':
c = camera(t[i]['arrivalTime'] -
t[i]['arrivalTime'].mean(),
cmap='Spectral',
ax=ax)
# mark_pixel(t[i]['shower'], color='k', linewidth=2.5)
ax.axis('off')
cb = fig.colorbar(c)
cb.set_label(label=r'$t-\bar{t}$ / ns', fontsize=16)
else:
c = camera(t[i]['photoncharge'], cmap='viridis', ax=ax)
ax.axis('off')
cb = fig.colorbar(c)
cb.set_label(label=r'Number of Photons', fontsize=16)
#mark_pixel(t[i]['shower'], color=(128/255, 186/255, 38/255), linewidth=2.5)
# mark_pixel(t[i]['shower'], color=(128/255, 186/255, 38/255), linewidth=2.5)
pdf.savefig(fig)
ax.cla()
plt.close(fig)
