class MigrateSingles(object):
"""Migrate singles to new table format
If the station has no slave *and* slave columns are all zero,
replace slave columns with `-1` to correctly represent missing slave.
"""
class HisparcSingle(tables.IsDescription):
event_id = tables.UInt32Col(pos=0)
timestamp = tables.Time32Col(pos=1)
mas_ch1_low = tables.Int32Col(dflt=-1, pos=2)
mas_ch1_high = tables.Int32Col(dflt=-1, pos=3)
mas_ch2_low = tables.Int32Col(dflt=-1, pos=4)
mas_ch2_high = tables.Int32Col(dflt=-1, pos=5)
slv_ch1_low = tables.Int32Col(dflt=-1, pos=6)
slv_ch1_high = tables.Int32Col(dflt=-1, pos=7)
slv_ch2_low = tables.Int32Col(dflt=-1, pos=8)
slv_ch2_high = tables.Int32Col(dflt=-1, pos=9)
def __init__(self, data):
self.data = data
self.singles_dtype = \
tables.description.dtype_from_descr(self.HisparcSingle)
self.network = HiSPARCNetwork(force_stale=True)
def migrate_table(self, table_path):
"""Migrate datatable to new format. Fix slave columns."""
logging.info('Migrating table: %s' % table_path)
group, table_name, sn = self._parse_path(table_path)
if table_name != 'singles':
logging.error('Table %s not `singles` skipping!' % table_path)
return None
tmp_table_name = '_t_%s' % table_name
try:
tmptable = self.data.create_table(group, tmp_table_name,
description=self.HisparcSingle)
except tables.NodeError:
logging.error('%s/_t_%s exists. Removing.' % (group, table_name))
self.data.remove_node(group, '_t_%s' % table_name)
tmptable = self.data.create_table(group, tmp_table_name,
description=self.HisparcSingle)
table = self.data.get_node(table_path)
data = table.read()
data = data.astype(self.singles_dtype)
if not self._has_slave(sn):
data = self._mark_slave_columns_as_missing(data)
tmptable.append(data)
tmptable.flush()
self.data.rename_node(table, 'singles_old')
self.data.rename_node(tmptable, 'singles')
def _parse_path(self, path):
""" '/cluster/s501/singles' ---> '/cluster/s501' 'singles', 501 """
group, table_name = tables.path.split_path(path)
re_number = re.compile('[0-9]+$')
numbers = [int(re_number.search(group).group())]
sn = numbers[-1]
return group, table_name, sn
def _has_slave(self, sn):
"""Return True if station (sn) has slave (4 detectors)"""
try:
n_detectors = len(self.network.get_station(sn).detectors)
except AttributeError:
logging.error('No information in HiSPARCNetwork() for sn %d' % sn)
n_detectors = 4
return n_detectors == 4
def _mark_slave_columns_as_missing(self, table):
"""Replace slave columns with `-1`"""
cols = ['slv_ch1_low', 'slv_ch2_low', 'slv_ch1_high', 'slv_ch2_high']
for col in cols:
if not np.all(table[col] == 0):
logging.error("Slave columns are not all zero. "
"Leaving data untouched!")
return table
n = len(table)
for col in cols:
table[col] = n * [-1]
logging.debug("Set all slave columns to `-1`.")
return table
def get_coincidence_count(close_pairs):
network = HiSPARCNetwork(force_stale=True)
distances = {4: [], 6: [], 8: []}
distance_errors = {4: [], 6: [], 8: []}
coincidence_rates = {4: [], 6: [], 8: []}
interval_rates = {4: [], 6: [], 8: []}
coincidence_rate_errors = {4: [], 6: [], 8: []}
pairs = {4: [], 6: [], 8: []}
for pair in pbar(close_pairs, show=True):
path = DATAPATH % tuple(pair)
if not os.path.exists(path):
continue
# Do not plot points for stations with known issues
bad_stations = [22, 507, 1001, 2103, 13007, 20001, 20002, 20003]
if pair[0] in bad_stations or pair[1] in bad_stations:
continue
with tables.open_file(path, 'r') as data:
try:
total_exposure = data.get_node_attr('/', 'total_exposure')
distance = network.calc_distance_between_stations(*pair)
n_rate = data.get_node_attr('/', 'n_rate')
interval_rate = data.get_node_attr('/', 'interval_rate')
n_coincidences = data.get_node_attr('/', 'n_coincidences')
except AttributeError:
# print 'failed reading attributes', pair
continue
if not n_coincidences:
continue
if n_coincidences < 5:
# Exclude pairs with very few coincidences
continue
n = (len(network.get_station(pair[0]).detectors) +
len(network.get_station(pair[1]).detectors))
distances[n].append(distance)
# Distance error due to unknown detector locations or moving stations
if pair[0] in NO_LAYOUT and pair[1] in NO_LAYOUT:
gps_layout_error = 20
elif pair[0] in NO_LAYOUT or pair[1] in NO_LAYOUT:
gps_layout_error = 10
else:
gps_layout_error = 3
distance_error = [
abs(d - distance) + gps_layout_error
for d in min_max_distance_pair(pair)
]
if distance_error[0] > distance:
distance_error[0] = distance - 1e-15
distance_errors[n].append(distance_error)
coincidence_rates[n].append(n_rate)
interval_rates[n].append(interval_rate)
err = sqrt(n_coincidences + 1) / total_exposure
# Prevent plotting issue due to log scale
rate = n_rate
if err > rate:
err = rate - 1e-15
coincidence_rate_errors[n].append(err)
pairs[n].append(pair)
return (distances, coincidence_rates, interval_rates, distance_errors,
coincidence_rate_errors, pairs)