def main(ts, ns):
station = Station(501)
traces = station.event_trace(ts, ns, True)
dr = DataReduction()
reduced_traces, o = dr.reduce_traces(array(traces).T, return_offset=True)
reduced_traces = reduced_traces.T
plot = Plot()
t = arange(len(traces[0])) * 2.5
for i, trace in enumerate(traces):
plot.plot(t, trace, linestyle='%s, thin' % COLORS[i], mark=None)
plot.draw_vertical_line(o * 2.5, 'gray')
plot.draw_vertical_line((o + len(reduced_traces[0])) * 2.5, 'gray')
plot.set_axis_options('line join=round')
plot.set_xlabel(r'Event time [\si{\ns}]')
plot.set_ylabel(r'Signal strength [ADCcounts]')
plot.set_xlimits(t[0], t[-1])
plot.save_as_pdf('raw_traces_%d_%d' % (ts, ns))
t = arange(o, o + len(reduced_traces[0])) * 2.5
for i, trace in enumerate(reduced_traces):
plot.plot(t, trace, linestyle='%s, thin' % COLORS[i], mark=None)
plot.set_axis_options('line join=round')
plot.set_xlabel(r'Event time [\si{\ns}]')
plot.set_ylabel(r'Signal strength [ADCcounts]')
plot.set_xlimits(t[0], t[-1])
plot.save_as_pdf('reduced_traces_%d_%d' % (ts, ns))
def determine_dt_for_pair(stations):
"""Determine and store dt for a pair of stations
:param ref_station: reference station number to use as refernece
:param station: station number to determine the dt for
"""
path = DATA_PATH + 'dt_ref%d_%d.h5' % stations
if os.path.exists(path):
print 'dt data already exists for %d-%d' % stations
return
ref_station, station = stations
try:
with tables.open_file(PAIR_DATAPATH % tuple(sorted(stations)),
'r') as data:
cq = CoincidenceQuery(data)
ref_detector_offsets = Station(ref_station).detector_timing_offset
detector_offsets = Station(station).detector_timing_offset
for dt0, dt1 in monthrange((2004, 1), (2015, 9)):
coins = cq.all(stations, start=dt0, stop=dt1, iterator=True)
coin_events = cq.events_from_stations(coins, stations)
ets, dt = determine_time_differences(coin_events, ref_station,
station,
ref_detector_offsets,
detector_offsets)
store_dt(ref_station, station, ets, dt)
except Exception as e:
print 'Failed for %d, %d' % stations
print e
return
def get_zenith_distribution():
station = Station(501)
counts = []
angles = []
for j in range(1, 13):
for i in range(1, 29):
try:
zenith_hist = station.zenith(2015, j, i)
except:
continue
total_counts = zenith_hist['counts'].sum()
if total_counts:
angles.extend(zenith_hist['angle'])
# Normalized
counts.extend(zenith_hist['counts'] / total_counts)
angles = array(angles).astype('float64')
counts = array(counts)
angles += 1.5 # bin edges to bin centers
angles_bins = arange(0, 91, 3)
mean_counts = binned_statistic(angles,
counts,
statistic='mean',
bins=angles_bins)[0]
std_counts = binned_statistic(angles,
counts,
statistic=std,
bins=angles_bins)[0]
angle_centers = (angles_bins[1:] + angles_bins[:-1]) / 2.
return angle_centers, mean_counts, std_counts
def coincidences_stations(station_numbers,
group_name='Specific stations',
date=None):
if date is None:
date = datetime.date(2016, 2, 1)
stations_with_data = []
cluster_groups = []
for station_id in station_numbers:
try:
info = Station(station_id)
except:
continue
if info.has_data(year=date.year, month=date.month, day=date.day):
cluster_groups.append(info.cluster().lower())
stations_with_data.append(station_id)
if len(stations_with_data) <= 1:
return
filepath = os.path.join(ESD_PATH, date.strftime('%Y/%-m/%Y_%-m_%-d.h5'))
with tables.open_file(filepath, 'r') as data:
coinc, event_tables = get_event_tables(data, cluster_groups,
stations_with_data)
windows, counts, n_events = find_n_coincidences(coinc, event_tables)
n_stations = len(stations_with_data)
plot_coinc_window(windows, counts, group_name, n_events, n_stations,
date)
return windows, counts
def get_all_configs():
stats = {}
for station in STATIONS:
stats[station] = {}
s = Station(station, force_stale=True)
for field in FIELDS:
timestamps = s.__getattribute__(field)['timestamp']
filter = (timestamps >= START_TS) & (timestamps < END_TS)
stats[station][field] = s.__getattribute__(field).compress(filter)
return stats
def get_latlon_coordinates(station_number):
"""Retrieve the GPS coordinates for a specific station
An exception is raised if the station does not have valid coordinates.
"""
station = Station(station_number)
gps_location = station.gps_location(START)
if gps_location['latitude'] == 0.:
raise Exception
return gps_location['latitude'], gps_location['longitude']
def get_latlon_coordinates(station_number):
"""Retrieve the GPS coordinates for a specific station
An exception is raised if the station does not have valid coordinates.
"""
station = Station(station_number)
gps_location = station.gps_location(START)
if gps_location['latitude'] == 0.:
raise Exception
return gps_location['latitude'], gps_location['longitude']
def get_active_stations():
""" return a list of all *active* station numbers in the HiSPAC Network"""
stations = Network(force_fresh=True).station_numbers()
for sn in stations:
s = Station(sn)
if s.info['active']:
yield sn
def get_data(station_numbers):
"""Read the eventtime csv files for the given station numbers"""
return {
number: Station(number, force_stale=True).event_time()
for number in station_numbers
}
def get_station_locations(country=None,
cluster=None,
subcluster=None,
station=None,
stations=None):
latitudes = []
longitudes = []
if station is not None:
station_numbers = [station]
elif stations is not None:
station_numbers = stations
else:
station_numbers = NETWORK.station_numbers(country=country,
cluster=cluster,
subcluster=subcluster)
for station_number in station_numbers:
location = Station(station_number, force_stale=True).gps_location()
if location['latitude'] == 0 or location['longitude'] == 0:
continue
latitudes.append(location['latitude'])
longitudes.append(location['longitude'])
return latitudes, longitudes
def find_overlaps():
with tables.open_file(DATA, 'r') as data:
events = data.root.s99.events
s = Station(99)
for i in range(events.nrows - 1):
if events[i +
1]['ext_timestamp'] - events[i]['ext_timestamp'] > 1e4:
continue
t1 = s.event_trace(events[i]['timestamp'],
events[i]['nanoseconds'], True)
t2 = s.event_trace(events[i + 1]['timestamp'],
events[i + 1]['nanoseconds'], True)
overlap = longest_overlap(t1[0], t2[0])
if overlap is not None:
print i, len(overlap) * 2.5, 'ns'
else:
print i, 'No overlap'
def get_station_end_timestamp(station, data):
"""Read all eventtime data into a dictionary"""
if Station(station, force_stale=True).info['active']:
return None
else:
# Start of day after last day with data
return data[station]['timestamp'][-1] + 3600
def get_traces(station_number, ts, ns, raw=False):
"""Retrieve the traces
:param station_number: number of the station to which the event belongs.
:param ts: timestamp of the event in seconds.
:param ns: subsecond part of the extended timestamp in nanoseconds.
:return: the traces.
"""
traces = Station(station_number).event_trace(ts, ns, raw)
return traces
def get_weather_locations():
latitudes = []
longitudes = []
station_numbers = [s['number'] for s in NETWORK.stations_with_weather()]
for station_number in station_numbers:
location = Station(station_number, force_stale=True).gps_location()
latitudes.append(location['latitude'])
longitudes.append(location['longitude'])
return latitudes, longitudes
def plot_station_offset_matrix():
n = len(STATIONS)
stations = {
station: Station(station, force_stale=True)
for station in STATIONS
}
for type in ['offset', 'error']:
plot = MultiPlot(n, n, width=r'.08\textwidth', height=r'.08\textwidth')
for i, station in enumerate(STATIONS):
for j, ref_station in enumerate(STATIONS):
splot = plot.get_subplot_at(i, j)
if i == n:
splot.show_xticklabels()
if station == ref_station:
splot.set_empty()
splot.set_label(r'%d' % station, location='center')
continue
offsets = stations[station].station_timing_offsets(ref_station)
bins = list(offsets['timestamp'])
bins += [bins[-1] + 86400]
splot.histogram(offsets[type], bins, linestyle='very thin')
splot.set_axis_options('line join=round')
plot_cuts_vertically(splot, stations, station, ref_station)
# Even/row rows/columns
for row in range(0, n, 2):
plot.set_yticklabels_position(row, n - 1, 'right')
plot.set_xticklabels_position(n - 1, row, 'bottom')
plot.show_yticklabels(row, n - 1)
#plot.show_xticklabels(n - 1, row)
for row in range(1, n, 2):
plot.set_yticklabels_position(row, 0, 'left')
plot.set_xticklabels_position(0, row, 'top')
plot.show_yticklabels(row, 0)
#plot.show_xticklabels(0, row)
if type == 'offset':
plot.set_ylimits_for_all(None, min=-70, max=70)
else:
plot.set_ylimits_for_all(None, min=0, max=10)
plot.set_xlimits_for_all(None,
min=datetime_to_gps(date(2011, 1, 1)),
max=datetime_to_gps(date.today()))
plot.set_xticks_for_all(None, YEARS_TICKS)
# plot.set_xtick_labels_for_all(YEARS_LABELS)
plot.set_xlabel(r'Date')
plot.set_ylabel(r'Station offset [\si{\ns}]')
plot.save_as_pdf('station_offsets_%s' % type)
def main(station_number=501, date=datetime.date(2016, 2, 1)):
filepath = os.path.join(ESD_PATH, date.strftime('%Y/%-m/%Y_%-m_%-d.h5'))
with tables.open_file(filepath, 'r') as data:
station = Station(station_number)
events = data.get_node(
'/hisparc/cluster_%s/station_%d' %
(station.cluster().lower(), station_number), 'events')
ext_timestamps = events.col('ext_timestamp')
ext_timestamps.sort()
difs = ext_timestamps[1:] - ext_timestamps[:-1]
print('Minimum: %d. Maximum: %d. n(diff < 100 us): %d' %
(min(difs), max(difs), len(numpy.where(difs < 1e5)[0])))
bins = numpy.logspace(2, 11)
plot = Plot('semilogx')
plot.histogram(*numpy.histogram(difs, bins=bins))
plot.set_xlabel(r'Time between subsequent triggers [\si{\ns}]')
plot.set_ylabel('Occurance')
plot.set_ylimits(min=0)
plot.set_xlimits(min(bins), max(bins))
plot.save_as_pdf('time_between_triggers_%d' % station_number)
def generate_json():
"""Get the API info data for each station"""
station_numbers = Network().station_numbers()
station_info = {}
for number in pbar(station_numbers):
try:
station = Station(number)
station_info[number] = station.info
except:
continue
return station_info
def reconstruct_events(data):
# Station 510
cluster = get_cluster()
station = cluster.get_station(STATIONS[1])
station_group = '/hisparc/cluster_amsterdam/station_%d' % station.number
rec_510 = ReconstructESDEvents(data,
station_group,
station,
overwrite=True,
progress=True)
rec_510.prepare_output()
rec_510.offsets = Station(station.number)
rec_510.reconstruct_directions()
rec_510.store_reconstructions()
rec_510.theta = array(rec_510.theta)
rec_510.phi = array(rec_510.phi)
# Station 507
for order in itertools.permutations(range(4), 4):
cluster = get_cluster()
station = cluster.get_station(STATIONS[0])
station_group = '/hisparc/cluster_amsterdam/station_%d' % station.number
station._detectors = [station.detectors[id] for id in order]
rec_507 = ReconstructESDEvents(data,
station_group,
station,
overwrite=True,
progress=True,
destination=REC_PATH % order)
rec_507.prepare_output()
rec_507.offsets = Station(station.number)
rec_507.reconstruct_directions(
detector_ids=[order[0], order[1], order[3]])
rec_507.store_reconstructions()
def plot_traces(coincidence_events):
plot = Plot()
t0 = int(coincidence_events[0][1]['ext_timestamp'])
tick_labels = []
tick_positions = []
for i, station_event in enumerate(coincidence_events):
station_number, event = station_event
station = Station(station_number)
traces = station.event_trace(event['timestamp'], event['nanoseconds'])
start_trace = (int(event['ext_timestamp']) - t0) - event['t_trigger']
t = arange(start_trace, start_trace + (2.5 * len(traces[0])), 2.5)
t = insert(t, 0, -20000)
t = append(t, 20000)
# trace = array(traces).sum(0)
for j, trace in enumerate(traces):
if max(trace) <= 10:
trace = array(trace)
else:
trace = array(trace) / float(max(trace)) * 100
trace = insert(trace, 0, 0)
trace = append(trace, 0)
plot.plot(t,
trace + (100 * j) + (500 * i),
mark=None,
linestyle=COLORS[j])
tick_labels.append(station_number)
tick_positions.append(500 * i)
plot.set_yticks(tick_positions)
plot.set_ytick_labels(tick_labels)
plot.set_xlimits(min=-250, max=1300)
plot.set_xlabel('t [\si{n\second}]')
plot.set_ylabel('Signal strength')
plot.save_as_pdf('traces_%d' % t0)
def plot_traces(event, station):
s = Station(station)
plot = Plot()
traces = s.event_trace(event['timestamp'], event['nanoseconds'], raw=True)
for j, trace in enumerate(traces):
t = arange(0, (2.5 * len(traces[0])), 2.5)
plot.plot(t, trace, mark=None, linestyle=COLORS[j])
n_peaks = event['n_peaks']
plot.set_title('%d - %d' % (station, event['ext_timestamp']))
plot.set_label('%d ' * 4 % tuple(n_peak for n_peak in n_peaks))
plot.set_xlabel('t [\si{n\second}]')
plot.set_ylabel('Signal strength')
plot.set_xlimits(min=0, max=2.5 * len(traces[0]))
plot.set_ylimits(min=150, max=500) # max=2 ** 12
plot.draw_horizontal_line(253, linestyle='gray')
plot.draw_horizontal_line(323, linestyle='gray')
plot.draw_horizontal_line(event['baseline'][0] + 20, linestyle='thin,gray')
plot.draw_horizontal_line(event['baseline'][1] + 20,
linestyle='thin,red!40!black')
plot.draw_horizontal_line(event['baseline'][2] + 20,
linestyle='thin,green!40!black')
plot.draw_horizontal_line(event['baseline'][3] + 20,
linestyle='thin,blue!40!black')
plot.save_as_pdf('traces_%d_%d' % (station, event['ext_timestamp']))
def get_histogram_for_station_on_date(station_id, date, did):
"""Return a histogram of the spectrum of a station on a date.
:return n, bins: histogram counts and bins, as obtained using
``numpy.histogram``.
"""
data = Station(station_id).pulse_integral(date.year, date.month, date.day)
bins = list(data['pulseintegral'])
bins.append(bins[-1] + (bins[-1] - bins[-2]))
bins = np.array(bins)
n = data['pi%d' % (did + 1)]
return n, bins
def normalize_event_rates(data, station_numbers):
"""Normalize event rates using the number of detectors
Number per hour is divided by the expected number of events per hour for a
station with a certain number of detectors.
So after this a '1.3' would be on average 30% more events per hour than the
expected number of events per hour for such a station.
"""
scaled_data = data.copy()
for i, s in enumerate(station_numbers):
n = Station(s).n_detectors()
if n == 2:
scaled_data[i] /= 1200.
elif n == 4:
scaled_data[i] /= 2500.
scaled_data = np.where(scaled_data > 2., 2., scaled_data)
return scaled_data
def get_offsets():
"""Setup nested dictionary with station timing offsets
The result looks like this:
{reference1: {station1: func, station2: func}, reference2: {...}, ...}
Get the offset (and error) between 501-502 on timestamp 1425168000 using:
offsets[501][502](1425168000)
# (6.1, 0.72)
"""
offsets = {
ref: {
s: partial(Station(s, force_stale=True).station_timing_offset,
reference_station=ref)
for s in STATIONS if not s == ref
}
for ref in STATIONS
}
return offsets
def main():
"""Demo the MPV finder with actual data."""
today = datetime.date.today()
yesterday = today - datetime.timedelta(days=1)
station_ids = get_station_ids_with_data(yesterday)
for station in station_ids:
plt.figure()
for did in range(Station(station).n_detectors()):
n, bins = get_histogram_for_station_on_date(station, yesterday,
did)
find_mpv = FindMostProbableValueInSpectrum(n, bins)
mpv, is_fitted = find_mpv.find_mpv()
plt.plot((bins[:-1] + bins[1:]) / 2., n, c=COLORS[did])
lines = ['dotted', 'solid']
plt.axvline(mpv + did * (bins[1] - bins[0]) / 20.,
c=COLORS[did], ls=lines[is_fitted])
plt.title(station)
plt.xlim(0, bins[len(bins)/2])
plt.yscale('log')
def get_latlon_coordinates(station_number):
station = Station(station_number)
gps_location = station.gps_location(START)
if gps_location['latitude'] == 0.:
raise Exception
return gps_location['latitude'], gps_location['longitude']
def get_offsets_dict():
offsets = {
s: Station(s).detector_timing_offset
for s in Network().station_numbers()
}
return offsets
def get_eventtime_data(station):
"""Get eventtime data"""
return Station(station, force_stale=True).event_time()
from pprint import PrettyPrinter
from random import sample
from sapphire import HiSPARCStations, Station
from sapphire.analysis.direction_reconstruction import CoincidenceDirectionReconstruction
# SN = [501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511]
SN = [501, 502, 503, 505, 506, 508, 510, 511]
cluster = HiSPARCStations(SN, force_stale=True)
crec = CoincidenceDirectionReconstruction(cluster)
offsets = {sn: Station(sn, force_stale=True) for sn in SN}
ref_sn = 501
station_number = 502
ts0 = 1461542400
print crec.determine_best_offsets(SN, ts0, offsets)
print offsets[station_number].station_timing_offset(ref_sn, ts0)
print offsets[station_number].detector_timing_offset(ts0)
offs = crec.determine_best_offsets([sn for sn in sample(SN, len(SN))], ts0,
offsets)
reference = offs[ref_sn][1]
offs = {sn: round(off[1] - reference, 1) for sn, off in offs.items()}
pp = PrettyPrinter()
pp.pprint(offs)
def get_station_locations(station):
"""All GPS locations for a single station"""
locations = Station(station, force_stale=True).gps_locations
return locations['latitude'], locations['longitude']
return
if len(gps_locations) < 2:
return
for p1, p2 in itertools.combinations(gps_locations, 2):
d = distance(p1, p2)
if d > .25:
print station.station, d
break
def distance(s1, s2):
"""
returns distance in km
"""
R = 6371 # km Radius of earth
d_lat = numpy.radians(s2['latitude'] - s1['latitude'])
d_lon = numpy.radians(s2['longitude'] - s1['longitude'])
a = (numpy.sin(d_lat / 2)**2 + numpy.cos(numpy.radians(s1['latitude'])) *
numpy.cos(numpy.radians(s2['latitude'])) * numpy.sin(d_lon / 2)**2)
c = 2 * numpy.arctan2(numpy.sqrt(a), numpy.sqrt(1 - a))
distance = R * c
return distance
if __name__ == "__main__":
for sn in Network().station_numbers():
station = Station(sn)
detect_problems(station)
import tables
from numpy import array, histogram, histogram2d, linspace
from artist import MultiPlot
from sapphire import Station
from sapphire.analysis.process_traces import MeanFilter, TraceObservables
from sapphire.publicdb import download_data
from sapphire.utils import pbar
DATA_PATH = '/Users/arne/Datastore/intergral_filter/data.h5'
STATION = 505
GROUP = '/s%d' % STATION
API_STATION = Station(STATION)
COLORS = ['black', 'red', 'green', 'blue']
def get_data():
"""Ensure data is downloaded and available"""
if not os.path.exists(DATA_PATH):
with tables.open_file(DATA_PATH, 'w') as data:
start = datetime.datetime(2014, 6, 10)
end = datetime.datetime(2014, 6, 11)
download_data(data, GROUP, STATION, start, end, get_blobs=True)
def get_traces_from_api(station, event):
return retrieve_traces(station, event['timestamp'], event['nanoseconds'])
plot.draw_horizontal_line(baseline, linestyle='densely dashed, gray')
plot.draw_horizontal_line(baseline + BASELINE_THRESHOLD,
linestyle='densely dotted, gray')
plot.draw_horizontal_line(max(trace), linestyle='densely dashed, gray')
# plot.set_ylimits(0)
plot.set_xlabel(r'Trace time [\si{\ns}]')
plot.set_ylabel(r'Signal strength [ADC]')
plot.save_as_pdf('integral')
def plot_filtered(filtered_trace):
plot = Plot()
time = arange(0, len(filtered_trace) * 2.5, 2.5)
plot.plot(time, filtered_trace, mark=None, linestyle='const plot')
plot.set_xlabel(r'Trace time [\si{\ns}]')
plot.set_ylabel(r'Signal strength [ADC]')
plot.save_as_pdf('mean_filter')
if __name__ == "__main__":
if 'trace' not in globals():
s = Station(510)
plot_integral(trace, baseline)
trace, baseline, integral, pulseheight, filtered_trace = get_trace(s)
plot_filtered(filtered_trace)
print '%d ADC, %d ADC.sample, %d ADC' % (baseline, integral, pulseheight)