graph = PolarPlot(use_radians=True)
times = generate_discrete_times(station, detector_ids=ids)
detectors = [station.detectors[id].get_coordinates() for id in ids]
x, y, z = zip(*detectors)
theta, phi = itertools.izip(*(dirrec.reconstruct_common((0, ) + t, x, y, z)
for t in times))
thetaa = [t for t in theta if not np.isnan(t)]
phia = [p for p in phi if not np.isnan(p)]
graph.scatter(phia, thetaa, markstyle='mark size=.5pt')
graph.set_ylimits(0, np.pi / 2)
graph.set_yticks([0, np.pi / 6, np.pi / 3, np.pi / 2])
graph.set_ytick_labels([
r'$0$', r'$\frac{1}{6}\pi$', r'$\frac{2}{6}\pi$', r'$\frac{1}{2}\pi$'
])
graph.set_ylabel('Zenith [rad]')
graph.set_xlabel('Azimuth [rad]')
graph.save_as_pdf('discrete_directions_%d_%s' %
(station.number, '_'.join(str(i) for i in ids)))
if __name__ == '__main__':
dirrec = DirectAlgorithmCartesian3D
station = HiSPARCStations([STATION]).get_station(STATION)
for combo in itertools.combinations(range(4), 3):
reconstruct_for_detectors(station, combo, dirrec)
import itertools
import tables
from numpy import abs, arange, array, histogram, std
from scipy.optimize import curve_fit
from artist import MultiPlot
from sapphire import HiSPARCStations
DATA_PATH = '/Users/arne/Datastore/station_offsets/'
SIM_PATH = '/Users/arne/Datastore/expected_dt/test_station_dt_spa.h5'
SPA_STAT = [501, 502, 503, 504, 505, 506, 508, 509, 510, 511]
CLUSTER = HiSPARCStations(SPA_STAT)
DAY = 86400.
HALF_DAY = DAY / 2
WEEK = 7 * DAY
FORTNIGHT = 2 * WEEK
XWEEK = 3 * WEEK
MONTH = 30 * DAY
QUARTER = 3 * MONTH
HALFYEAR = 6 * MONTH
YEAR = 365 * DAY
def get_station_dt(data, station):
table = data.get_node('/s%d' % station)
return table
import tables
from numpy import arange
from artist import Plot
from sapphire import CoincidenceQuery, HiSPARCStations
from sapphire.analysis.event_utils import relative_detector_arrival_times
from sapphire.utils import distance_between
COIN_DATA = '/Users/arne/Datastore/esd_coincidences/coincidences_n7_120101_140801.h5'
DETECTOR_IDS = [0, 1, 2, 3]
STATIONS = [501, 502, 503, 504, 505, 506, 508, 509]
CLUSTER = HiSPARCStations(STATIONS)
OFFSETS = {
501: [-1.10338, 0.0000, 5.35711, 3.1686],
502: [-8.11711, -8.5528, -8.72451, -9.3388],
503: [-22.9796, -26.6098, -22.7522, -21.8723],
504: [-15.4349, -15.2281, -15.1860, -16.5545],
505: [-21.6035, -21.3060, -19.6826, -25.5366],
506: [-20.2320, -15.8309, -14.1818, -14.1548],
508: [-26.2402, -24.9859, -24.0131, -23.2882],
509: [-24.8369, -23.0218, -20.6011, -24.3757]
}
COLORS = {
501: 'black',
502: 'red!80!black',
503: 'blue!80!black',
504: 'green!80!black',
505: 'orange!80!black',
506: 'pink!80!black',
def distances_netherlands():
sn = Network(force_stale=True).station_numbers(country=0)
cluster = HiSPARCStations(sn, force_stale=True, skip_missing=True)
distances_stations(cluster, name='_netherlands')
from sapphire import CoincidenceQuery, HiSPARCStations, Station
from sapphire.analysis import event_utils
from sapphire.transformations import geographic
COIN_DATA = '/Users/arne/Datastore/esd_coincidences/sciencepark_n11_150701_151105.h5'
# OFFSETS = {501: [-1.10338, 0.0000, 5.35711, 3.1686],
# 502: [-8.11711, -8.5528, -8.72451, -9.3388],
# 503: [-22.9796, -26.6098, -22.7522, -21.8723],
# 504: [-15.4349, -15.2281, -15.1860, -16.5545],
# 505: [-21.6035, -21.3060, -19.6826, -25.5366],
# 506: [-20.2320, -15.8309, -14.1818, -14.1548],
# 508: [-26.2402, -24.9859, -24.0131, -23.2882],
# 509: [-24.8369, -23.0218, -20.6011, -24.3757]}
DETECTOR_IDS = [0, 1, 2, 3]
STATIONS = range(501, 512)
CLUSTER = HiSPARCStations(STATIONS)
COLORS = ['black', 'red!80!black', 'green!80!black', 'blue!80!black']
def make_map(cluster=CLUSTER):
latitudes = []
longitudes = []
for station in cluster.stations:
for detector in station.detectors:
latitude, longitude, _ = detector.get_lla_coordinates()
latitudes.append(latitude)
longitudes.append(longitude)
map = Map(
(min(latitudes), min(longitudes), max(latitudes), max(longitudes)))
return map
def get_cluster():
"""Get latest position from API"""
return HiSPARCStations(STATIONS)
COLORS = ['black', 'red', 'green', 'blue']
def plot_detectors(cluster):
station = cluster.stations[0]
detectors = station.detectors
timestamps = set(station.timestamps).union(detectors[0].timestamps)
plot = Plot()
for timestamp in sorted(timestamps):
cluster.set_timestamp(timestamp)
for i in range(4):
x, y = detectors[i].get_xy_coordinates()
plot.scatter([x], [y], mark='*', markstyle=COLORS[i])
x, y = station.get_xy_coordinates()
plot.scatter([x], [y], markstyle='purple')
# print timestamp, gps_to_datetime(timestamp), x, y
plot.set_xlabel(r'Easting [\si{\meter}]')
plot.set_ylabel(r'Northing [\si{\meter}]')
plot.set_axis_equal()
plot.save_as_pdf('locations_%d' % station.number)
if __name__ == "__main__":
for sn in range(501, 512):
# print sn
cluster = HiSPARCStations([sn], force_stale=True)
plot_detectors(cluster)
def distance_between_stations(s1, s2):
cluster = HiSPARCStations([s1, s2], force_stale=True)
xyz = [
array(s.calc_center_of_mass_coordinates()) for s in cluster.stations
]
return distance(*xyz)
import warnings
import tables
from sapphire import GroundParticlesSimulation, HiSPARCStations
if __name__ == "__main__":
with warnings.catch_warnings():
warnings.simplefilter("ignore")
cluster = HiSPARCStations(
[501, 502, 503, 504, 505, 506, 508, 509, 510, 511])
corsika_path = 'corsika.h5'
with tables.open_file('result.h5', 'w') as result:
sim = GroundParticlesSimulation(corsikafile_path=corsika_path,
max_core_distance=1006.58,
cluster=cluster,
datafile=result,
output_path='/',
N=100000,
seed=153957,
progress=False)
sim.run()
sim.finish()
def merge(stations, output = None, orig_stations=None, directory='.', verbose=True,
overwrite=False, reconstruct=False, save_coordinates=False,
only_original=False, coincidences=1, cluster=None, photontimes=False):
"""
Merges the simulation data from individual 'the_simulation.h5' files from the
core.* directories inside a certain directory
PS: Deze functie is ook een beetje een draak en heeft ook nodig onderhoud nodig.
Feitelijk gebeurt er niet heel veel: je leest alleen alle the_simulation.h5
bestanden uit en combineert ze. Het is dan waarschijnlijker ook makkelijker als je
zelf even iets klust.
:param stations: List of stations to use
:param output: The output file, by default main_data[503_505_...].h5
:param orig_stations: If there are more stations than those listed in the parameter
in the simulation files you need to list them here. By
default this is None, indicating that the stations are the
same
:param directory: The directory to look in, by default the current directory
:param only_original: If True only look for directories in the format coreXX,
else look in every directory
:param verbose: If True print more output statements
:param overwrite: If True overwrite the Sapphire reconstruction in the
individual the_simulation.h5 files
:param reconstruct: If True reconstruct directions using Sapphire
:param save_coordinates:If True store the impact coordinates of the first particle
that hit the detector (requires a version of sapphire that
saves this information)
:param coincidences: Minimum number of coincidences to include
:param cluster: A Cluster object, if None creates a Cluster object itself
using the list of stations provided.
:param photontimes: If True stores the photontimes histogram. Also adds the
individual timings to the photontimes, so traces are correctly
timed relative to each other.
:return: nothing
"""
# process parameters
STATIONS = stations
if orig_stations is not None:
ORIG_STATIONS = orig_stations
else:
ORIG_STATIONS = STATIONS
ORIG_LENGTH = len(ORIG_STATIONS)
if output is not None:
output_file = output
else:
output_file = 'main_data_%s.h5' % str(STATIONS).replace(', ', '_')
# if the length of the original stations is 1 then sapphire will not have included
# a coincidence table, so just grab all the events from the one station
if ORIG_LENGTH==1:
IGNORE_COINCIDENCES = True
else:
IGNORE_COINCIDENCES = False
combined_regex = "(" + ")|(".join([str(a) for a in STATIONS]) + ")"
N = len(STATIONS)
if cluster is None:
cluster = HiSPARCStations(STATIONS)
if only_original:
core_re = re.compile(r"core.*\d$")
else:
core_re = re.compile(r"core.*")
dirs = [os.path.join(directory, o) for o in os.listdir(directory) if
os.path.isdir(os.path.join(directory, o)) and core_re.match(o) is not None]
# The pytables class that describes the table to store in the h5 file
class Traces(tables.IsDescription):
id = tables.Int32Col()
N = tables.Int32Col()
azimuth = tables.Float32Col()
zenith = tables.Float32Col()
traces = tables.Int16Col(shape=(len(STATIONS), 4, 80))
energy = tables.Float32Col()
timings = tables.Float32Col(shape=(len(STATIONS), 4))
pulseheights = tables.Int16Col(shape=(len(STATIONS), 4))
x = tables.Float32Col()
y = tables.Float32Col()
z = tables.Float32Col()
azimuth_rec = tables.Float32Col(dflt=np.nan)
zenith_rec = tables.Float32Col(dflt=np.nan)
core_distance = tables.Float32Col()
core_position = tables.Float32Col(shape=(2,))
photontimes = tables.Float32Col(shape=(len(STATIONS), 4, 80,))
arrivaltimes_particles = tables.Float32Col(shape=(len(STATIONS), 4))
if save_coordinates:
inslag_coordinates = tables.Float32Col((4,2))
n_electron_muons = tables.Int16Col(shape=4)
with tables.open_file(output_file, mode='w',
title='Collected data from %s' % STATIONS) as collected_traces:
# create the table /traces/traces
group = collected_traces.create_group('/', 'traces',
'Traces with azimuth and zenith information')
table = collected_traces.create_table(group, 'Traces', Traces, 'Traces')
row = table.row
# keep track of numbers in order to print those at the end
total = 0
throwing_away = 0
# loop over all core* directories
for d in tqdm(dirs):
# wrap in try except block, because sometimes a core* dir exists,
# but without the_simulation.h5 file
try:
template = '%s/the_simulation.h5' % d
# open only in append mode if reconstructing direction and if there is
# a sim.py.e1091201 file in the directory, which indicates that the
# simulation is done running and we can safely open the file in 'a' mode
list_of_files = os.listdir(d)
re_sim = re.compile('sim\.py\.e[0-9]*')
reconstruct_local = False
if reconstruct:
if np.count_nonzero([re_sim.match(x) for x in list_of_files])>0:
reconstruct_local = True
if reconstruct_local:
fmode = 'a'
else:
fmode = 'r'
with tables.open_file(template, fmode) as data:
if IGNORE_COINCIDENCES and reconstruct_local:
# only possible if there is 1 station (for now)
station_path = '/cluster_simulations/station_%s/' % STATIONS[0]
station = STATIONS[0]
rec = ReconstructSimulatedEvents(data, station_path, station,
verbose=False, overwrite=overwrite,
progress=False)
try:
rec.reconstruct_and_store()
except:
if verbose:
print('Already reconstructed')
recs = data.get_node(station_path).reconstructions
if IGNORE_COINCIDENCES: # create one entry for every event in every
# station
for station in data.root.cluster_simulations:
if photontimes:
photontimes_table = station.photontimes
for station_event in station.events:
timings_station = np.array(
[station_event['t1'], station_event['t2'],
station_event['t3'], station_event['t4']])
# due to a bug in the simulation sometimes a timing
# of -999 is included, so filter these out (bug is
# fixed now, so should no longer be neccessary unless
# when working with old data)
if filter_timings(timings_station):
# create empty arrays to fill
trace = np.zeros([4, 80], dtype=np.float32)
timings = np.zeros([4], dtype=np.float32)
pulseheights = np.zeros([4], dtype=np.int16)
# fill using data from h5 file
trace_local = station_event['traces']
trace_local[trace_local < -MAX_VOLTAGE] = -MAX_VOLTAGE
trace[:, :] = trace_local
zenith = station_event['zenith']
azimuth = station_event['azimuth']
energy = station_event['shower_energy']
distance_core = station_event['core_distance']
if np.count_nonzero(np.isnan(timings_station))>0:
print(timings_station)
# remove the -999 timings and set to 0
timings_station[timings_station < 0] = 0
timings[:] = timings_station
pulseheights_local = station_event['pulseheights']
pulseheights_local[pulseheights_local >
MAX_VOLTAGE] = MAX_VOLTAGE
pulseheights[:] = pulseheights_local
if np.count_nonzero(
(pulseheights>((4096*0.57/10e3)*1e3-1))
&
(pulseheights<((4096*0.57/10e3)*1e3+1))
)>0:
throwing_away += 1
continue
# write to new h5 file
# do some magic to recreate timings and shift
# photontimes (this is really a bit of a pointless
# exercise, since you should just do this right in
# the simulation anyway)
particle_timings = np.zeros((4,))
earliest_particle = np.inf
non_zero_idx = []
for i, (t, t0) in enumerate(zip(trace, timings)):
old_trigger_delay = 0
for i_local, value in enumerate(t):
if value < -30.0:
old_trigger_delay = i_local * 2.5
break
else:
particle_timings[i] = np.nan
continue
non_zero_idx.append(i)
particle_timings[i] = timings[i] - old_trigger_delay
if particle_timings[i] < earliest_particle:
earliest_particle = particle_timings[i]
particle_timings -= np.nanmin(particle_timings)
row['arrivaltimes_particles'] = particle_timings
if photontimes:
row_photontimes = np.zeros((4,80))
for i, idx in enumerate(station_event['photontimes_idx']):
if np.isnan(particle_timings[i]): #there is
# no incident particle or the
# particles that were incidident did
# not produce a sufficiently high trace
# to trigger
continue
pt = photontimes_table[idx] + \
particle_timings[i]
local_hist, _ = np.histogram(pt,
bins=np.linspace(0,200,81))
row_photontimes[i,:] = local_hist
row['photontimes'] = row_photontimes
row['traces'] = trace
row['N'] = 1
row['azimuth'] = azimuth
row['zenith'] = zenith
row['energy'] = energy
row['timings'] = timings
# convert zenith and azimuth to x y z
x, y, z = azimuth_zenith_to_cartestian(zenith, azimuth)
row['x'] = x
row['y'] = y
row['z'] = z
if reconstruct:
row['azimuth_rec'] = recs.col('azimuth')[station_event['event_id']]
row['zenith_rec'] = recs.col('zenith')[station_event['event_id']]
row['pulseheights'] = pulseheights
row['core_distance'] = distance_core
row['id'] = total
if save_coordinates:
row['inslag_coordinates'] = station_event[
'coordinates']
row['n_electron_muons'] = \
[station_event["n_electrons1"] +
station_event["n_muons1"],
station_event["n_electrons2"] +
station_event["n_muons2"],
station_event["n_electrons3"] +
station_event["n_muons3"],
station_event["n_electrons4"] +
station_event["n_muons4"]]
row.append()
total += 1
else:
throwing_away += 1
data.close()
else:
if reconstruct and reconstruct_local:
rec = ReconstructSimulatedCoincidences(data,
destination='reconstructions',
overwrite=overwrite,
progress=False)
rec.reconstruct_and_store()
recs = data.get_node('/coincidences/reconstructions')
# recreating coincidences, so per coincidence a list of the
# traces etc. per station
# for every incoming shower the coincidences are saved for some
# reason, but only if there is a hit the timestamp>0
# WARNING: I have not used this for a long time,
# so it probably is broken (or does not the same things as with
# the single station). Basically do not use.
for coin in data.root.coincidences.coincidences.where(
'timestamp>0'):
if coin['N'] >= coincidences:
trace = np.zeros([len(STATIONS), 4, 80], dtype=np.float32)
timings = np.zeros([len(STATIONS), 4], dtype=np.float32)
pulseheights = np.zeros([len(STATIONS), 4], dtype=np.int16)
# the c_index maps the coincidences to the events in
# the format [[station_id, event_id],...]. You then use
# the
# s_index table to look up the path to the station
# using the station_id
c_index = data.root.coincidences.c_index[coin['id']]
for station, event_idx in c_index:
station_path = data.root.coincidences.s_index[station].\
decode('UTF-8')
# make sure the path exists
if re.search(combined_regex, station_path) is not None:
station_event = data.get_node(station_path, 'events')[
event_idx]
station = STATIONS.index(ORIG_STATIONS[station])
trace_local = station_event['traces']
trace_local[trace_local<-MAX_VOLTAGE] = -MAX_VOLTAGE
trace[station, :, :] = trace_local
zenith = station_event['zenith']
azimuth = station_event['azimuth']
energy = station_event['shower_energy']
timings_station = np.array(
[station_event['t1'], station_event['t2'],
station_event['t3'], station_event['t4']])
timings_station[timings_station < 0] = 0
timings[station, :] = timings_station
pulseheights_local = station_event['pulseheights']
pulseheights_local[pulseheights_local>MAX_VOLTAGE] = MAX_VOLTAGE
pulseheights[station, :] = station_event['pulseheights']
# due to stupidity I set the cap to 4096*0.57/10e3
# mV instead of 4096*0.57/1e3 ...
if np.count_nonzero(
(pulseheights>((4096*0.57/10e3)*1e3-1))
&
(pulseheights<((4096*0.57/10e3)*1e3+1))
)>0:
throwing_away += 1
continue
row['traces'] = trace
row['N'] = coin['N']
row['azimuth'] = azimuth
row['zenith'] = zenith
row['energy'] = energy
row['timings'] = timings
x, y, z = azimuth_zenith_to_cartestian(zenith, azimuth)
row['x'] = x
row['y'] = y
row['z'] = z
if reconstruct:
row['azimuth_rec'] = recs.col('azimuth')[coin['id']]
row['zenith_rec'] = recs.col('zenith')[coin['id']]
row['pulseheights'] = pulseheights
row['id'] = total
row['core_position'] = [coin['x'], coin['y']]
row.append()
total += 1
else:
throwing_away += 1
except Exception as e:
if verbose:
print('Error occurred in %s' % (d))
print(e)
table.flush()
print('Total entries: %d' % total)
print('Thrown away: %s' % throwing_away)
from pprint import PrettyPrinter
from random import sample
from sapphire import HiSPARCStations, Station
from sapphire.analysis.direction_reconstruction import CoincidenceDirectionReconstruction
# SN = [501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511]
SN = [501, 502, 503, 505, 506, 508, 510, 511]
cluster = HiSPARCStations(SN, force_stale=True)
crec = CoincidenceDirectionReconstruction(cluster)
offsets = {sn: Station(sn, force_stale=True) for sn in SN}
ref_sn = 501
station_number = 502
ts0 = 1461542400
print crec.determine_best_offsets(SN, ts0, offsets)
print offsets[station_number].station_timing_offset(ref_sn, ts0)
print offsets[station_number].detector_timing_offset(ts0)
offs = crec.determine_best_offsets([sn for sn in sample(SN, len(SN))], ts0,
offsets)
reference = offs[ref_sn][1]
offs = {sn: round(off[1] - reference, 1) for sn, off in offs.items()}
pp = PrettyPrinter()
pp.pprint(offs)
from artist import Plot
from sapphire import HiSPARCStations
from sapphire.transformations.clock import datetime_to_gps
STATIONS = [502, 503, 504, 505, 506, 508, 509, 510]
DAY = 86400
HALF_DAY = DAY / 2
WEEK = 7 * DAY
FORTNIGHT = 2 * WEEK
XWEEK = 3 * WEEK
MONTH = 30 * DAY
YEAR = 365 * DAY
SPA_STAT = [501, 502, 503, 504, 505, 506, 508, 509, 510]
CLUSTER = HiSPARCStations(SPA_STAT)
DATA_PATH = '/Users/arne/Datastore/station_offsets/'
def get_station_dt(data, station):
table = data.get_node('/s%d' % station)
return table
if __name__ == '__main__':
t_start = datetime_to_gps(datetime(2010, 1, 1))
t_end = datetime_to_gps(datetime(2015, 4, 1))
for i, station in enumerate(STATIONS, 1):
with tables.open_file(DATA_PATH + 'dt_ref501_%d.h5' % station, 'r') as data:
def get_cluster():
"""Get latest position from API"""
return HiSPARCStations(STATIONS, skip_missing=True)
def get_cluster():
return HiSPARCStations(STATIONS)
def do_simulation_small():
with tables.open_file(RESULT_DATA_SMALL, 'w') as data:
cluster = HiSPARCStations([501], force_stale=True)
sim = ModGroundParticlesSimulation(CORSIKA_DATA_SMALL, 100, cluster, data,
N=4000, progress=True)
sim.run()
