def process_data(self, filename, max_events=None):
#Open output hdf5 file
f = tables.open_file(self.output_path, mode="a", title="Output File")
# Open Root File
root_file = VARootFile(filename)
self.write_metadata(f, root_file)
selected_tels, num_tel = self.select_telescopes(root_file)
arr_info = root_file.get_array_info()
if not f.__contains__('/Array_Info'):
arr_table = f.create_table(f.root, 'Array_Info', row_types.Array,
("Table of array data"))
arr_row = arr_table.row
for tel_type in selected_tels:
for tel_id in selected_tels[tel_type]:
arr_row["tel_id"] = arr_info[tel_id]['tel_id']
arr_row["tel_x"] = arr_info[tel_id]['tel_x']
arr_row["tel_y"] = arr_info[tel_id]['tel_y']
arr_row["tel_z"] = arr_info[tel_id]['tel_z']
arr_row["tel_type"] = arr_info[tel_id]['tel_type']
arr_row["run_array_direction"] = arr_info[tel_id][
'run_array_direction']
arr_row.append()
if not f.__contains__('/Telescope_Info'):
tel_table = f.create_table(f.root, 'Telescope_Info', row_types.Tel,
("Table of telescope data"))
descr = tel_table.description._v_colobjects
descr2 = descr.copy()
if (self.one_D_image_oversampled):
logger.debug('Save dummy pixel pos')
descr2["pixel_pos"] = tables.Float32Col(shape=(2, 54 * 54))
tel_table2 = f.create_table(f.root, 'temp', descr2,
"Table of telescope data")
tel_table.attrs._f_copy(tel_table2)
tel_table.remove()
tel_table2.move(f.root, 'Telescope_Info')
tel_row = tel_table2.row
# add units to table attributes
random_tel_type = "VTS"
random_tel_id = 0
tel_table2.attrs.tel_pos_units = str("aru")
tel_row['tel_type'] = "VTS"
posx, posy = np.meshgrid(np.arange(0, 54, dtype='float32'),
np.arange(0, 54, dtype='float32'))
tel_row['pixel_pos'] = [posx.ravel(), posy.ravel()]
tel_row.append()
else:
logger.debug('Save real pixel pos')
descr2["pixel_pos"] = tables.Float32Col(shape=(2, 499))
tel_table2 = f.create_table(f.root, 'temp', descr2,
"Table of telescope data")
tel_table.attrs._f_copy(tel_table2)
tel_table.remove()
tel_table2.move(f.root, 'Telescope_Info')
tel_row = tel_table2.row
# add units to table attributes
random_tel_type = "VTS"
random_tel_id = 0
tel_table2.attrs.tel_pos_units = str("mm")
tel_row['tel_type'] = "VTS"
pos = np.loadtxt(data_dir + '/pixel_position.txt')
tel_row['pixel_pos'] = [pos[:, 0], pos[:, 1]]
tel_row.append()
#create event table
if not f.__contains__('/Event_Info'):
table = f.create_table(f.root, 'Event_Info', row_types.Event,
"Table of Event metadata")
descr = table.description._v_colobjects
descr2 = descr.copy()
if self.storage_mode == 'tel_type':
for tel_type in selected_tels:
descr2[tel_type + '_indices'] = tables.Int32Col(
shape=(len(selected_tels[tel_type])))
elif self.storage_mode == 'tel_id':
descr2["indices"] = tables.Int32Col(shape=(num_tel))
table2 = f.create_table(f.root, 'temp', descr2, "Table of Events")
table.attrs._f_copy(table2)
table.remove()
table2.move(f.root, 'Event_Info')
#add units to table attributes
table2.attrs.core_pos_units = 'm'
table2.attrs.h_first_int_units = 'm'
table2.attrs.mc_energy_units = 'TeV'
table2.attrs.alt_az_units = 'rad'
#create image tables
for tel_type in selected_tels:
if self.img_mode == '2D':
img_width = self.IMAGE_SHAPES[tel_type][0]
img_length = self.IMAGE_SHAPES[tel_type][1]
if self.img_dim_order == 'channels_first':
array_shape = (1, img_width, img_length)
elif self.img_dim_order == 'channels_last':
array_shape = (img_width, img_length, 1)
np_type = np.dtype(
(np.dtype(self.img_dtypes[tel_type]), array_shape))
columns_dict = {
"image": tables.Col.from_dtype(np_type),
"event_index": tables.Int32Col()
}
elif self.img_mode == '1D':
array_shape = (image.TEL_NUM_PIXELS_OVER_SAMPLED[tel_type],
) if self.one_D_image_oversampled else (
image.TEL_NUM_PIXELS[tel_type], )
np_type = np.dtype(
(np.dtype(self.img_dtypes[tel_type]), array_shape))
columns_dict = {
"image_charge": tables.Col.from_dtype(np_type),
"event_index": tables.Int32Col(),
"image_peak_times": tables.Col.from_dtype(np_type)
}
description = type('description', (tables.IsDescription, ),
columns_dict)
if self.storage_mode == 'tel_type':
if not f.__contains__('/' + tel_type):
table = f.create_table(
f.root, tel_type, description,
"Table of {} images".format(tel_type))
#append blank image at index 0
image_row = table.row
if self.img_mode == '2D':
image_row['image'] = self.trace_converter.convert(
np.zeros(500))
elif self.img_mode == '1D':
shape = (image.TEL_NUM_PIXELS_OVER_SAMPLED[tel_type],
) if self.one_D_image_oversampled else (
image.TEL_NUM_PIXELS[tel_type], )
image_row['image_charge'] = np.zeros(
shape, dtype=self.img_dtypes[tel_type])
image_row['image_peak_times'] = np.zeros(
shape, dtype=self.img_dtypes[tel_type])
image_row['event_index'] = -1
image_row.append()
table.flush()
elif self.storage_mode == 'tel_id':
for tel_id in selected_tels[tel_type]:
if not f.__contains__('/T' + str(tel_id)):
table = f.create_table(
f.root, 'T' + str(tel_id), description,
"Table of T{} images".format(str(tel_id)))
#append blank image at index 0
image_row = table.row
if self.img_mode == '2D':
image_row['image'] = self.trace_converter.convert(
np.zeros(500))
elif self.img_mode == '1D':
shape = (
image.TEL_NUM_PIXELS_OVER_SAMPLED[tel_type],
) if self.one_D_image_oversampled else (
image.TEL_NUM_PIXELS[tel_type], )
image_row['image_charge'] = np.zeros(
shape, dtype=self.img_dtypes[tel_type])
image_row['image_peak_times'] = np.zeros(
shape, dtype=self.img_dtypes[tel_type])
image_row['event_index'] = -1
image_row.append()
table.flush()
# specify calibration and other processing options
logger.info("Processing events...")
event_count = 0
passing_count = 0
if (max_events is None):
source = root_file.read_st2_calib_channel_charge(
tels=[i - 1 for i in selected_tels[tel_type]], cleaning=None)
else:
source = root_file.read_st2_calib_channel_charge(
tels=[i - 1 for i in selected_tels[tel_type]],
cleaning=None,
stop_event=max_events)
for i, simData, event, tzero, triggeredTels in source:
if ((max_events is not None) and (event_count > max_events)):
break
event_count += 1
table = f.root.Event_Info
table.flush()
event_row = table.row
event_index = table.nrows
if self.storage_mode == 'tel_type':
tel_index_vectors = {
tel_type: []
for tel_type in selected_tels
}
elif self.storage_mode == 'tel_id':
all_tel_index_vector = []
for tel_type in selected_tels.keys():
for tel_id in sorted(selected_tels[tel_type]):
if self.storage_mode == 'tel_type':
index_vector = tel_index_vectors[tel_type]
elif self.storage_mode == 'tel_id':
index_vector = all_tel_index_vector
if (self.force_all_telescopes):
triggeredTels = [1, 2, 3, 4]
if tel_id in triggeredTels:
pixel_vector = event[tel_id - 1, :]
timing_vector = tzero[tel_id - 1, :]
logger.debug(
'Storing image from tel_type {} ({} pixels)'.
format(tel_type, len(pixel_vector)))
if self.storage_mode == 'tel_type':
table = eval('f.root.{}'.format(tel_type))
elif self.storage_mode == 'tel_id':
table = eval('f.root.T{}'.format(tel_id))
next_index = table.nrows
image_row = table.row
imgs = self.trace_converter.convert(pixel_vector)
time_imgs = self.trace_converter.convert(
timing_vector) * 4
if self.img_mode == '2D':
image_row['image'] = imgs
elif self.img_mode == '1D':
if (self.one_D_image_oversampled):
image_row['image_charge'] = imgs.reshape(
array_shape).copy()
image_row[
'image_peak_times'] = time_imgs.reshape(
array_shape).copy()
else:
image_row[
'image_charge'] = pixel_vector[:499].copy(
)
image_row[
'image_peak_times'] = timing_vector[:
499].copy(
)
image_row["event_index"] = event_index
image_row.append()
index_vector.append(next_index)
table.flush()
else:
index_vector.append(0)
if self.storage_mode == 'tel_type':
for tel_type in tel_index_vectors:
event_row[tel_type +
'_indices'] = tel_index_vectors[tel_type]
elif self.storage_mode == 'tel_id':
event_row['indices'] = all_tel_index_vector
event_row['event_number'] = i
event_row['run_number'] = simData.fRunNum
event_row['particle_id'] = simData.fCORSIKAParticleID
event_row['core_x'] = simData.fCoreEastM
event_row['core_y'] = simData.fCoreSouthM * -1
event_row['h_first_int'] = 0
event_row['mc_energy'] = simData.fEnergyGeV / 1000.
event_row['alt'] = simData.fPrimaryZenithDeg * np.pi * 180.
event_row['az'] = simData.fPrimaryAzimuthDeg * np.pi * 180.
event_row.append()
table.flush()
f.root.Event_Info.flush()
total_num_events = f.root.Event_Info.nrows
f.close()
logger.info("{} events read in file".format(event_count))
logger.info("{} total events in output file.".format(total_num_events))
logger.info("Done!")
class Test(tb.IsDescription):
ngroup = tb.Int32Col(pos=1)
ntable = tb.Int32Col(pos=2)
nrow = tb.Int32Col(pos=3)
class Descr(tables.IsDescription):
var1 = tables.StringCol(itemsize=4, shape=(), dflt='', pos=0)
var2 = tables.BoolCol(shape=(), dflt=False, pos=1)
var3 = tables.Int32Col(shape=(), dflt=0, pos=2)
var4 = tables.Float64Col(shape=(), dflt=0.0, pos=3)
class TimeSeriesGrid(tb.IsDescription):
sat_name = tb.StringCol(10, pos=1)
ref_time = tb.StringCol(10, pos=2)
date = tb.StringCol(10, pos=3)
year = tb.Int32Col(pos=4)
month = tb.Int32Col(pos=5)
class TimeSeriesGrid(tb.IsDescription):
satname = tb.StringCol(20, pos=1)
time1 = tb.Int32Col(pos=2)
time2 = tb.Int32Col(pos=3)
class Obs(tables.IsDescription):
#date_time = tables.Int32Col(indexed=True, pos=1)
#year = tables.Int16Col(indexed=True, pos=2)
#yday = tables.Int16Col(indexed=True, pos=3)
#hour = tables.Int8Col(indexed=True, pos=4)
date_time = tables.Int32Col(pos=1)
year = tables.Int16Col(pos=2)
yday = tables.Int16Col(pos=3)
hour = tables.Int8Col(pos=4)
source = tables.StringCol(1, pos=5)
type = tables.StringCol(6, pos=6)
wind_dir = tables.Int16Col(pos=7)
wdir_type = tables.StringCol(1, pos=8)
wind_spd = tables.Float32Col(pos=9)
cig = tables.UInt16Col(pos=10)
cig_tool = tables.StringCol(1, pos=11)
cavok = tables.StringCol(1, pos=12)
vis = tables.Int32Col(pos=13)
vis_var_code = tables.StringCol(1, pos=14)
temp = tables.Float32Col(pos=15)
dewt = tables.Float32Col(pos=16)
pres = tables.Float32Col(pos=17)
prec_period = tables.UInt8Col(shape=p_dim, pos=18)
prec_depth = tables.Float32Col(shape=p_dim, pos=19)
prec_code = tables.UInt8Col(shape=p_dim, pos=20)
dur_code = tables.UInt8Col(pos=21)
char_code = tables.StringCol(1, pos=22)
disc_code = tables.UInt8Col(pos=23)
water_dep = tables.Int16Col(pos=24)
snow_dim = tables.Int16Col(pos=25)
snow_code = tables.UInt8Col(pos=26)
liq_dim = tables.Float32Col(pos=27)
liq_code = tables.UInt8Col(pos=28)
snow_period = tables.UInt8Col(shape=p_dim, pos=29)
snow_accu = tables.Int16Col(shape=p_dim, pos=30)
accu_cond = tables.UInt8Col(shape=p_dim, pos=31)
pres_wx_code = tables.UInt8Col(pos=32)
pt_mwx_code = tables.UInt8Col(shape=o_dim, pos=33)
pt_mwx_pd = tables.UInt8Col(shape=o_dim, pos=34)
pt_awx_code = tables.UInt8Col(shape=o_dim, pos=35)
pt_awx_pd = tables.UInt8Col(shape=o_dim, pos=36)
rw_vis_dir = tables.Int16Col(pos=37)
rw_code = tables.StringCol(1, pos=38)
rw_vis_dim = tables.Int16Col(pos=39)
cov_code = tables.UInt8Col(shape=c_dim, pos=40)
bs_hi_dim = tables.Int32Col(shape=c_dim, pos=41)
cloud_code = tables.UInt8Col(shape=c_dim, pos=42)
cov_sum_st_code = tables.UInt8Col(shape=c_dim, pos=43)
cov_sum_code = tables.UInt8Col(shape=c_dim, pos=44)
cov_sum_st_dim = tables.Int32Col(shape=c_dim, pos=45)
cov_sum_char_code = tables.UInt8Col(shape=c_dim, pos=46)
total_cov_code = tables.UInt8Col(pos=47)
total_opa_code = tables.UInt8Col(pos=48)
low_cov_code = tables.UInt8Col(pos=49)
low_cld_gen_code = tables.UInt8Col(pos=50)
low_cld_dim = tables.Int32Col(pos=51)
mid_cld_gen_code = tables.UInt8Col(pos=52)
hi_cld_gen_code = tables.UInt8Col(pos=53)
st_cov_code = tables.UInt8Col(shape=c_dim, pos=54)
st_cld_tp_hi = tables.Int32Col(shape=c_dim, pos=55)
st_cld_type = tables.UInt8Col(shape=c_dim, pos=56)
st_cld_tp_code = tables.UInt8Col(shape=c_dim, pos=57)
sun_dur = tables.Int16Col(pos=58)
hail_size = tables.Float32Col(pos=59)
g2s_code = tables.UInt8Col(pos=60)
mint_period = tables.Float32Col(pos=61)
mint = tables.Float32Col(pos=62)
x_tp_period = tables.Float32Col(shape=o_dim, pos=63)
x_tp_code = tables.UInt8Col(shape=o_dim, pos=64)
x_tp = tables.Float32Col(shape=o_dim, pos=65)
altimeter = tables.Float32Col(pos=66)
st_pres = tables.Float32Col(pos=67)
pres_tr = tables.UInt8Col(pos=68)
pres_chg_3h = tables.Float32Col(pos=69)
pres_chg_24h = tables.Float32Col(pos=70)
isobar_code = tables.UInt8Col(pos=71)
isobar_dim = tables.Int16Col(pos=72)
wx_vic_code = tables.UInt8Col(shape=d_dim, pos=73)
pres_wxm_code = tables.UInt8Col(shape=d_dim, pos=74)
sup_wd_code = tables.UInt8Col(shape=s_dim, pos=75)
sup_wd_prd = tables.UInt8Col(shape=s_dim, pos=76)
sup_wd_spd = tables.Float32Col(shape=s_dim, pos=77)
wd_gust = tables.Float32Col(pos=78)
class unit_descriptor(tables.IsDescription):
electrode_number = tables.Int32Col()
single_unit = tables.Int32Col()
regular_spiking = tables.Int32Col()
fast_spiking = tables.Int32Col()
class RunInfo(tb.IsDescription):
run_number = tb.Int32Col(shape=(), pos=0)
class Distance(tables.IsDescription):
frame = tables.Int32Col(pos=0)
distance = tables.Float64Col(pos=1)
class SymbolCount(tb.IsDescription):
symbol_root = tb.StringCol(6, pos=1)
count = tb.Int32Col(pos=2) # short integer
date = tb.Int32Col(pos=3)
class EventInfo(tb.IsDescription):
evt_number = tb.Int32Col(shape=(), pos=0)
timestamp = tb.UInt64Col(shape=(), pos=1)
class FillModel(pt.IsDescription):
timestamp = pt.Time64Col()
quantity = pt.Int32Col()
cashChange = pt.Float32Col()
commission = pt.Float32Col()
impactCost = pt.Float32Col()
class DECONV_PARAM(tb.IsDescription):
N_BASELINE = tb.Int32Col(pos=0)
THR_TRIGGER = tb.Int16Col(pos=1)
ACCUM_DISCHARGE_LENGTH = tb.Int16Col(pos=2)
class TrialData(tables.IsDescription):
trial_num = tables.Int32Col()
trigger = tables.StringCol(26)
response = tables.StringCol(26)
plot_trigger = tables.Int8Col()
plot_response = tables.Int8Col()
class Particle(tb.IsDescription):
name = tb.StringCol(16, pos=1) # 16-character String
lati = tb.Int32Col(pos=2) # integer
longi = tb.Int32Col(pos=3) # integer
pressure = tb.Float32Col(pos=4) # float (single-precision)
temperature = tb.Float64Col(pos=5) # double (double-precision)
class Small(tb.IsDescription):
var1 = tb.StringCol(itemsize=4)
var2 = tb.Int32Col()
var3 = tb.Float64Col()
var4 = tb.BoolCol()
class TrialData(tables.IsDescription):
trial_num = tables.Int32Col()
session = tables.Int32Col()
class ProcessEvents(object):
"""Process HiSPARC events to obtain several observables.
This class can be used to process a set of HiSPARC events and adds a
few observables like particle arrival time and number of particles in
the detector to a copy of the event table.
"""
processed_events_description = {
'event_id': tables.UInt32Col(pos=0),
'timestamp': tables.Time32Col(pos=1),
'nanoseconds': tables.UInt32Col(pos=2),
'ext_timestamp': tables.UInt64Col(pos=3),
'data_reduction': tables.BoolCol(pos=4),
'trigger_pattern': tables.UInt32Col(pos=5),
'baseline': tables.Int16Col(pos=6, shape=4, dflt=-1),
'std_dev': tables.Int16Col(pos=7, shape=4, dflt=-1),
'n_peaks': tables.Int16Col(pos=8, shape=4, dflt=-1),
'pulseheights': tables.Int16Col(pos=9, shape=4, dflt=-1),
'integrals': tables.Int32Col(pos=10, shape=4, dflt=-1),
'traces': tables.Int16Col(pos=11, shape=(4, 80), dflt=-1),
'event_rate': tables.Float32Col(pos=12),
't1': tables.Float32Col(pos=13, dflt=-1),
't2': tables.Float32Col(pos=14, dflt=-1),
't3': tables.Float32Col(pos=15, dflt=-1),
't4': tables.Float32Col(pos=16, dflt=-1),
'n1': tables.Float32Col(pos=17, dflt=-1),
'n2': tables.Float32Col(pos=18, dflt=-1),
'n3': tables.Float32Col(pos=19, dflt=-1),
'n4': tables.Float32Col(pos=20, dflt=-1),
't_trigger': tables.Float32Col(pos=21, dflt=-1)
}
def __init__(self, data, group, source=None, progress=True):
"""Initialize the class.
:param data: the PyTables datafile
:param group: the group containing the station data. In normal
cases, this is simply the group containing the events table.
:param source: the name of the events table. Default: None,
meaning the default name 'events'.
:param progress: if True show a progressbar while copying and
processing events.
"""
self.data = data
self.group = data.get_node(group)
self.source = self._get_source(source)
self.progress = progress
self.limit = None
def process_and_store_results(self,
destination=None,
overwrite=False,
limit=None):
"""Process events and store the results.
:param destination: name of the table where the results will be
written. The default, None, corresponds to 'events'.
:param overwrite: if True, overwrite previously obtained results.
:param limit: the maximum number of events that will be stored.
The default, None, corresponds to no limit.
"""
self.limit = limit
self._check_destination(destination, overwrite)
self._clean_events_table()
self._create_results_table()
self._store_results_from_traces()
self._store_number_of_particles()
self._move_results_table_into_destination()
def get_traces_for_event(self, event):
"""Return the traces from an event.
:param event: a row from the events table.
:return: the traces: an array of pulseheight values.
"""
traces = [
list(self._get_trace(idx)) for idx in event['traces'] if idx >= 0
]
# Make traces follow NumPy conventions
traces = np.array(traces).T
return traces
def get_traces_for_event_index(self, idx):
"""Return the traces from event #idx.
:param idx: the index number of the event.
:return: the traces: an array of pulseheight values.
"""
event = self.source[idx]
return self.get_traces_for_event(event)
def _get_source(self, source):
"""Return the table containing the events.
:param source: the *name* of the table. If None, this method will
try to find the original events table, even if the events were
previously processed.
:return: table object
"""
if source is None:
if '_events' in self.group:
source = self.group._events
else:
source = self.group.events
else:
source = self.data.get_node(self.group, source)
return source
def _check_destination(self, destination, overwrite):
"""Check if the destination is valid"""
if destination == '_events':
raise RuntimeError("The _events table is reserved for internal "
"use. Choose another destination.")
elif destination is None:
destination = 'events'
# If destination == source, source will be moved out of the way. Don't
# worry. Otherwise, destination may not exist or will be overwritten
if self.source.name != destination:
if destination in self.group and not overwrite:
raise RuntimeError("I will not overwrite previous results "
"(unless you specify overwrite=True)")
self.destination = destination
def _clean_events_table(self):
"""Clean the events table.
Remove duplicate events and sort the table by ext_timestamp.
"""
events = self.source
enumerated_timestamps = list(enumerate(events.col('ext_timestamp')))
enumerated_timestamps.sort(key=operator.itemgetter(1))
unique_sorted_ids = self._find_unique_row_ids(enumerated_timestamps)
new_events = self._replace_table_with_selected_rows(
events, unique_sorted_ids)
self.source = new_events
self._normalize_event_ids(new_events)
def _find_unique_row_ids(self, enumerated_timestamps):
"""Find the unique row_ids from enumerated timestamps."""
prev_timestamp = 0
unique_sorted_ids = []
for row_id, timestamp in enumerated_timestamps:
if timestamp != prev_timestamp:
# event is unique, so add it
unique_sorted_ids.append(row_id)
prev_timestamp = timestamp
return unique_sorted_ids
def _replace_table_with_selected_rows(self, table, row_ids):
"""Replace events table with selected rows.
:param table: original table to be replaced.
:param row_ids: row ids of the selected rows which should go in
the destination table.
"""
tmptable = self.data.create_table(self.group,
't__events',
description=table.description)
selected_rows = table.read_coordinates(row_ids)
tmptable.append(selected_rows)
tmptable.flush()
self.data.rename_node(tmptable, table.name, overwrite=True)
return tmptable
def _normalize_event_ids(self, events):
"""Normalize event ids.
After sorting, the event ids no longer correspond to the row
number. This can complicate finding the row id of a particular
event. This method will replace the event_ids of all events by
the row id.
:param events: the events table to normalize.
"""
row_ids = list(range(len(events)))
events.modify_column(column=row_ids, colname='event_id')
def _create_results_table(self):
"""Create results table containing the events."""
self._tmp_events = self._create_empty_results_table()
self._copy_events_into_table()
def _create_empty_results_table(self):
"""Create empty results table with correct length."""
if self.limit:
length = self.limit
else:
length = len(self.source)
if '_t_events' in self.group:
self.data.remove_node(self.group, '_t_events')
table = self.data.create_table(self.group,
'_t_events',
self.processed_events_description,
expectedrows=length)
for _ in range(length):
table.row.append()
table.flush()
return table
def _copy_events_into_table(self):
table = self._tmp_events
source = self.source
for col in pbar(source.colnames, show=self.progress):
table.modify_column(stop=self.limit,
colname=col,
column=getattr(source.cols, col)[:self.limit])
table.flush()
def _store_results_from_traces(self):
table = self._tmp_events
timings = self.process_traces()
# Assign values to full table, column-wise.
for idx in range(4):
col = 't%d' % (idx + 1)
table.modify_column(column=timings[:, idx], colname=col)
table.flush()
def process_traces(self):
"""Process traces to yield pulse timing information."""
if self.limit is not None:
events = self.source.iterrows(stop=self.limit)
else:
events = self.source
timings = self._process_traces_from_event_list(events,
length=self.limit)
return timings
def _process_traces_from_event_list(self, events, length=None):
"""Process traces from a list of events.
This is the method looping over all events.
:param events: an iterable of the events
:param length: an indication of the number of events, for use as a
progress bar. Optional.
"""
result = []
for event in pbar(events, length=length, show=self.progress):
timings = self._reconstruct_time_from_traces(event)
result.append(timings)
timings = np.array(result)
return timings
def _reconstruct_time_from_traces(self, event):
"""Reconstruct arrival times for a single event.
This method loops over the traces.
:param event: row from the events table.
:return: arrival times in the detectors relative to trace start
in ns.
"""
timings = []
for baseline, pulseheight, trace_idx in zip(event['baseline'],
event['pulseheights'],
event['traces']):
if pulseheight < 0:
# retain -1, -999 status flags in timing
timings.append(pulseheight)
elif pulseheight < ADC_THRESHOLD:
timings.append(-999)
else:
trace = self._get_trace(trace_idx)
timings.append(
self._reconstruct_time_from_trace(trace, baseline))
timings = [
time * ADC_TIME_PER_SAMPLE if time not in ERR else time
for time in timings
]
return timings
def _get_trace(self, idx):
"""Returns a trace given an index into the blobs array.
Decompress a trace from the blobs array.
:param idx: index into the blobs array
:return: iterator over the pulseheight values
"""
blobs = self._get_blobs()
try:
trace = zlib.decompress(blobs[idx]).decode('utf-8').split(',')
except zlib.error:
trace = (zlib.decompress(
blobs[idx][1:-1]).decode('utf-8').split(','))
if trace[-1] == '':
del trace[-1]
trace = (int(x) for x in trace)
return trace
def _get_blobs(self):
return self.group.blobs
def _reconstruct_time_from_trace(self, trace, baseline):
"""Reconstruct time of measurement from a trace.
This method is doing the hard work.
:param trace: array containing pulseheight values.
:param baseline: baseline of the trace.
:return: index in trace for arrival time of first particle.
"""
threshold = baseline + ADC_THRESHOLD
value = self.first_above_threshold(trace, threshold)
return value
@staticmethod
def first_above_threshold(trace, threshold):
"""Find the first element in the list equal or above threshold
If no element matches the condition -999 will be returned.
:param trace: iterable trace.
:param threshold: value the trace has to be greater or equal to.
:return: index in trace where a value is greater or equal to
threshold.
"""
return next((i for i, x in enumerate(trace) if x >= threshold), -999)
def _store_number_of_particles(self):
"""Store number of particles in the detectors.
Process all pulseintegrals from the events and estimate the number
of particles in each detector.
"""
table = self._tmp_events
n_particles = self._process_pulseintegrals()
for idx in range(4):
col = 'n%d' % (idx + 1)
table.modify_column(column=n_particles[:, idx], colname=col)
table.flush()
def _process_pulseintegrals(self):
"""Find MPVs using pulseintegrals to estimate number of particles
:return: array with estimated number of particles per detector per
event.
"""
n_particles = []
integrals = self.source.col('integrals')
all_mpv = []
for detector_integrals in integrals.T:
if (detector_integrals < 0).all():
all_mpv.append(np.nan)
else:
n, bins = np.histogram(detector_integrals,
bins=np.linspace(0, 50000, 201))
find_mpv = FindMostProbableValueInSpectrum(n, bins)
mpv, is_fitted = find_mpv.find_mpv()
if is_fitted:
all_mpv.append(mpv)
else:
all_mpv.append(np.nan)
all_mpv = np.array(all_mpv)
for event in self.source[:self.limit]:
pulseintegrals = event['integrals']
# retain -1, -999 status flags
pulseintegrals = np.where(pulseintegrals >= 0,
pulseintegrals / all_mpv, pulseintegrals)
# if mpv fit failed, value is nan. Make it -999
pulseintegrals = np.where(np.isnan(pulseintegrals), -999,
pulseintegrals)
n_particles.append(pulseintegrals)
return np.array(n_particles)
def _move_results_table_into_destination(self):
if self.source.name == 'events':
self.source.rename('_events')
self.source = self.group._events
if self.destination in self.group:
self.data.remove_node(self.group, self.destination)
self._tmp_events.rename(self.destination)
def __repr__(self):
if not self.data.isopen:
return "<finished %s>" % self.__class__.__name__
else:
return ("%s(%r, %r, source=%r, progress=%r)" %
(self.__class__.__name__, self.data.filename,
self.group._v_pathname, self.source._v_pathname,
self.progress))
class PhenotypeValue(tables.IsDescription):
"""
Phenotype value class
"""
ecotype = tables.Int32Col()
value = tables.Float32Col()
class Record(tables.IsDescription):
col1 = tables.Int32Col()
col2 = tables.Int32Col()
col3 = tables.Float64Col()
col4 = tables.Float64Col()
# read one file to extract TF and Electrod info
TF = cf.Eph(AllEph[0]).TF
Electrodes = cf.Eph(AllEph[0]).Electrodes
AllData = H5.createEArray(DataGroup, 'All', tables.Float64Atom(),
(TF * Electrodes, 0))
GFPData = H5.createEArray(DataGroup, 'GFP', tables.Float64Atom(), (TF, 0))
# Reading EphFile dans store into Tables with EArray
for e in AllEph:
dat = cf.Eph(e)
AllData.append(dat.Data.reshape(np.array(dat.Data.shape).prod(), 1))
GFPData.append(dat.GFP.reshape(TF, 1))
ShapeOriginalData = H5.createArray('/', 'Shape', np.array(dat.Data.shape))
ModelParticle = {
'Name': tables.StringCol(40),
'Value': tables.Int32Col(shape=len(SubjectFactor)),
'Type': tables.StringCol(40)
}
InfoParticle = {}
for c in InfoDict:
InfoParticle[c] = tables.StringCol(256)
AnovaAllParticle = {
'StatEffect': tables.StringCol(40),
'P': tables.Float64Col(shape=(TF, Electrodes)),
'F': tables.Float64Col(shape=(TF, Electrodes))
}
AnovaGFPParticle = {
'StatEffect': tables.StringCol(40),
'P': tables.Float64Col(shape=(TF, 1)),
'F': tables.Float64Col(shape=(TF, 1))
class TriggerInfoData(tables.IsDescription):
EventID = tables.Int64Col(pos=0)
Sec = tables.Int32Col(pos=1)
NanoSec = tables.Int32Col(pos=2)
def main():
# Argument parser
parser = make_argparser()
parser.add_argument(
"--debug",
action="store_true",
help="Print debugging information",
)
parser.add_argument(
"--save_images",
action="store_true",
help="Save also all images",
)
parser.add_argument(
"--estimate_energy",
type=str2bool,
default=False,
help="Estimate the events' energy with a regressor from\
protopipe.scripts.build_model",
)
parser.add_argument("--regressor_dir",
type=str,
default="./",
help="regressors directory")
parser.add_argument(
"--regressor_config",
type=str,
default=None,
help="Configuration file used to produce regressor model")
args = parser.parse_args()
# Read configuration file
cfg = load_config(args.config_file)
try: # If the user didn't specify a site and/or and array...
site = cfg["General"]["site"]
array = cfg["General"]["array"]
except KeyError: # ...raise an error and exit.
print("\033[91m ERROR: make sure that both 'site' and 'array' are "
"specified in the analysis configuration file! \033[0m")
sys_exit(-1)
if args.infile_list:
filenamelist = []
for f in args.infile_list:
filenamelist += glob("{}/{}".format(args.indir, f))
filenamelist.sort()
else:
raise ValueError("don't know which input to use...")
if not filenamelist:
print("no files found; check indir: {}".format(args.indir))
sys_exit(-1)
else:
print("found {} files".format(len(filenamelist)))
# Get the IDs of the involved telescopes and associated cameras together
# with the equivalent focal lengths from the first event
allowed_tels, cams_and_foclens, subarray = prod3b_array(
filenamelist[0], site, array)
# keeping track of events and where they were rejected
evt_cutflow = CutFlow("EventCutFlow")
img_cutflow = CutFlow("ImageCutFlow")
preper = EventPreparer(
config=cfg,
subarray=subarray,
cams_and_foclens=cams_and_foclens,
mode=args.mode,
event_cutflow=evt_cutflow,
image_cutflow=img_cutflow,
)
# catch ctr-c signal to exit current loop and still display results
signal_handler = SignalHandler()
signal.signal(signal.SIGINT, signal_handler)
# Regressor information
regressor_method = cfg["EnergyRegressor"]["method_name"]
try:
estimation_weight = cfg["EnergyRegressor"]["estimation_weight"]
except KeyError:
estimation_weight = "STD"
# wrapper for the scikit-learn regressor
if args.estimate_energy is True:
# Read configuration file
regressor_config = load_config(args.regressor_config)
log_10_target = regressor_config["Method"]["log_10_target"]
regressor_files = (args.regressor_dir +
"/regressor_{cam_id}_{regressor}.pkl.gz")
reg_file = regressor_files.format(
**{
"mode": args.mode,
"wave_args": "mixed", # ToDo, control
"regressor": regressor_method,
"cam_id": "{cam_id}",
})
regressors = load_models(reg_file, cam_id_list=cams_and_foclens.keys())
# COLUMN DESCRIPTOR AS DICTIONARY
# Column descriptor for the file containing output training data."""
DataTrainingOutput = dict(
# ======================================================================
# ARRAY
obs_id=tb.Int16Col(dflt=1, pos=0),
event_id=tb.Int32Col(dflt=1, pos=1),
tel_id=tb.Int16Col(dflt=1, pos=2),
N_LST=tb.Int16Col(dflt=1, pos=3),
N_MST=tb.Int16Col(dflt=1, pos=4),
N_SST=tb.Int16Col(dflt=1, pos=5),
n_tel_reco=tb.FloatCol(dflt=1, pos=6),
n_tel_discri=tb.FloatCol(dflt=1, pos=7),
# ======================================================================
# DL1
hillas_intensity_reco=tb.Float32Col(dflt=1, pos=8),
hillas_intensity=tb.Float32Col(dflt=1, pos=9),
hillas_x_reco=tb.Float32Col(dflt=1, pos=10),
hillas_y_reco=tb.Float32Col(dflt=1, pos=11),
hillas_x=tb.Float32Col(dflt=1, pos=12),
hillas_y=tb.Float32Col(dflt=1, pos=13),
hillas_r_reco=tb.Float32Col(dflt=1, pos=14),
hillas_r=tb.Float32Col(dflt=1, pos=15),
hillas_phi_reco=tb.Float32Col(dflt=1, pos=16),
hillas_phi=tb.Float32Col(dflt=1, pos=17),
hillas_length_reco=tb.Float32Col(dflt=1, pos=18),
hillas_length=tb.Float32Col(dflt=1, pos=19),
hillas_width_reco=tb.Float32Col(dflt=1, pos=20),
hillas_width=tb.Float32Col(dflt=1, pos=21),
hillas_psi_reco=tb.Float32Col(dflt=1, pos=22),
hillas_psi=tb.Float32Col(dflt=1, pos=23),
hillas_skewness_reco=tb.Float32Col(dflt=1, pos=24),
hillas_skewness=tb.Float32Col(dflt=1, pos=25),
hillas_kurtosis=tb.Float32Col(dflt=1, pos=26),
hillas_kurtosis_reco=tb.Float32Col(dflt=1, pos=27),
leakage_intensity_width_1_reco=tb.Float32Col(dflt=np.nan, pos=28),
leakage_intensity_width_2_reco=tb.Float32Col(dflt=np.nan, pos=29),
leakage_intensity_width_1=tb.Float32Col(dflt=np.nan, pos=30),
leakage_intensity_width_2=tb.Float32Col(dflt=np.nan, pos=31),
concentration_cog=tb.Float32Col(dflt=np.nan, pos=32),
concentration_core=tb.Float32Col(dflt=np.nan, pos=33),
concentration_pixel=tb.Float32Col(dflt=np.nan, pos=34),
# The following are missing from current ctapipe DL1 output
# Not sure if it's worth to add them
hillas_ellipticity_reco=tb.FloatCol(dflt=1, pos=35),
hillas_ellipticity=tb.FloatCol(dflt=1, pos=36),
max_signal_cam=tb.Float32Col(dflt=1, pos=37),
pixels=tb.Int16Col(dflt=1, pos=38),
clusters=tb.Int16Col(dflt=-1, pos=39),
# ======================================================================
# DL2 - DIRECTION RECONSTRUCTION
impact_dist=tb.Float32Col(dflt=1, pos=40),
h_max=tb.Float32Col(dflt=1, pos=41),
alt=tb.Float32Col(dflt=np.nan, pos=42),
az=tb.Float32Col(dflt=np.nan, pos=43),
err_est_pos=tb.Float32Col(dflt=1, pos=44),
err_est_dir=tb.Float32Col(dflt=1, pos=45),
xi=tb.Float32Col(dflt=np.nan, pos=46),
offset=tb.Float32Col(dflt=np.nan, pos=47),
mc_core_x=tb.FloatCol(dflt=1, pos=48),
mc_core_y=tb.FloatCol(dflt=1, pos=49),
reco_core_x=tb.FloatCol(dflt=1, pos=50),
reco_core_y=tb.FloatCol(dflt=1, pos=51),
mc_h_first_int=tb.FloatCol(dflt=1, pos=52),
mc_x_max=tb.Float32Col(dflt=np.nan, pos=53),
is_valid=tb.BoolCol(dflt=False, pos=54),
good_image=tb.Int16Col(dflt=1, pos=55),
# ======================================================================
# DL2 - ENERGY ESTIMATION
true_energy=tb.FloatCol(dflt=1, pos=56),
reco_energy=tb.FloatCol(dflt=np.nan, pos=57),
reco_energy_tel=tb.Float32Col(dflt=np.nan, pos=58),
# ======================================================================
# DL1 IMAGES
# this is optional data saved by the user
# since these data declarations require to know how many pixels
# each saved image will have,
# we add them later on, right before creating the table
# We list them here for reference
# true_image=tb.Float32Col(shape=(1855), pos=56),
# reco_image=tb.Float32Col(shape=(1855), pos=57),
# cleaning_mask_reco=tb.BoolCol(shape=(1855), pos=58), # not in ctapipe
)
outfile = tb.open_file(args.outfile, mode="w")
outTable = {}
outData = {}
for i, filename in enumerate(filenamelist):
print("file: {} filename = {}".format(i, filename))
source = EventSource(input_url=filename,
allowed_tels=allowed_tels,
max_events=args.max_events)
# loop that cleans and parametrises the images and performs the
# reconstruction for each event
for (
event,
reco_image,
cleaning_mask_reco,
cleaning_mask_clusters,
true_image,
n_pixel_dict,
hillas_dict,
hillas_dict_reco,
leakage_dict,
concentration_dict,
n_tels,
max_signals,
n_cluster_dict,
reco_result,
impact_dict,
good_event,
good_for_reco,
) in preper.prepare_event(source,
save_images=args.save_images,
debug=args.debug):
if good_event:
xi = angular_separation(event.simulation.shower.az,
event.simulation.shower.alt,
reco_result.az, reco_result.alt)
offset = angular_separation(
event.pointing.array_azimuth,
event.pointing.array_altitude,
reco_result.az,
reco_result.alt,
)
# Impact parameter
reco_core_x = reco_result.core_x
reco_core_y = reco_result.core_y
# Height of shower maximum
h_max = reco_result.h_max
# Todo add conversion in number of radiation length,
# need an atmosphere profile
is_valid = True
else: # something went wrong and the shower's reconstruction failed
xi = np.nan * u.deg
offset = np.nan * u.deg
reco_core_x = np.nan * u.m
reco_core_y = np.nan * u.m
h_max = np.nan * u.m
reco_result.alt = np.nan * u.deg
reco_result.az = np.nan * u.deg
is_valid = False
reco_energy = np.nan
reco_energy_tel = dict()
# Not optimal at all, two loop on tel!!!
# For energy estimation
# Estimate energy only if the shower was reconstructed
if (args.estimate_energy is True) and is_valid:
weight_tel = np.zeros(len(hillas_dict.keys()))
energy_tel = np.zeros(len(hillas_dict.keys()))
for idx, tel_id in enumerate(hillas_dict.keys()):
# use only images that survived cleaning and
# parametrization
if not good_for_reco[tel_id]:
# bad images will get an undetermined energy
# this is a per-telescope energy
# NOT the estimated energy for the shower
reco_energy_tel[tel_id] = np.nan
continue
cam_id = source.subarray.tel[tel_id].camera.camera_name
moments = hillas_dict[tel_id]
model = regressors[cam_id]
############################################################
# GET FEATURES
############################################################
# Read feature list from model configutation file
features_basic = regressor_config["FeatureList"]["Basic"]
features_derived = regressor_config["FeatureList"][
"Derived"]
features = features_basic + list(features_derived)
# Create a pandas Dataframe with basic quantities
# This is needed in order to connect the I/O system of the
# model inputs to the in-memory computation of this script
data = pd.DataFrame({
"hillas_intensity": [moments.intensity],
"hillas_width": [moments.width.to("deg").value],
"hillas_length": [moments.length.to("deg").value],
"hillas_x": [moments.x.to("deg").value],
"hillas_y": [moments.y.to("deg").value],
"hillas_phi": [moments.phi.to("deg").value],
"hillas_r": [moments.r.to("deg").value],
"leakage_intensity_width_1_reco":
[leakage_dict[tel_id]['leak1_reco']],
"leakage_intensity_width_2_reco":
[leakage_dict[tel_id]['leak2_reco']],
"leakage_intensity_width_1":
[leakage_dict[tel_id]['leak1']],
"leakage_intensity_width_2":
[leakage_dict[tel_id]['leak2']],
"concentration_cog":
[concentration_dict[tel_id]['concentration_cog']],
"concentration_core":
[concentration_dict[tel_id]['concentration_core']],
"concentration_pixel":
[concentration_dict[tel_id]['concentration_pixel']],
"az": [reco_result.az.to("deg").value],
"alt": [reco_result.alt.to("deg").value],
"h_max": [h_max.value],
"impact_dist": [impact_dict[tel_id].to("m").value],
})
# Compute derived features and add them to the dataframe
for key, expression in features_derived.items():
data.eval(f'{key} = {expression}', inplace=True)
# features_img = np.array(
# [
# np.log10(moments.intensity),
# np.log10(impact_dict[tel_id].value),
# moments.width.value,
# moments.length.value,
# h_max.value,
# ]
# )
# sort features_to_use alphabetically to ensure order
# preservation with model.fit in protopipe.mva
features = sorted(features)
# Select the values for the full set of features
features_values = data[features].to_numpy()
############################################################
if estimation_weight == "STD":
# Get an array of trees
predictions_trees = np.array([
tree.predict(features_values)
for tree in model.estimators_
])
energy_tel[idx] = np.mean(predictions_trees, axis=0)
weight_tel[idx] = np.std(predictions_trees, axis=0)
else:
data.eval(f'estimation_weight = {estimation_weight}',
inplace=True)
energy_tel[idx] = model.predict(features_values)
weight_tel[idx] = data["estimation_weight"]
if log_10_target:
energy_tel[idx] = 10**energy_tel[idx]
weight_tel[idx] = 10**weight_tel[idx]
reco_energy_tel[tel_id] = energy_tel[idx]
reco_energy = np.sum(weight_tel * energy_tel) / sum(weight_tel)
else:
for idx, tel_id in enumerate(hillas_dict.keys()):
reco_energy_tel[tel_id] = np.nan
for idx, tel_id in enumerate(hillas_dict.keys()):
cam_id = source.subarray.tel[tel_id].camera.camera_name
if cam_id not in outData:
if args.save_images is True:
# we define and save images content here, to make it
# adaptive to different cameras
n_pixels = source.subarray.tel[
tel_id].camera.geometry.n_pixels
DataTrainingOutput["true_image"] = tb.Float32Col(
shape=(n_pixels), pos=56)
DataTrainingOutput["reco_image"] = tb.Float32Col(
shape=(n_pixels), pos=57)
DataTrainingOutput["cleaning_mask_reco"] = tb.BoolCol(
shape=(n_pixels), pos=58) # not in ctapipe
DataTrainingOutput[
"cleaning_mask_clusters"] = tb.BoolCol(
shape=(n_pixels), pos=58) # not in ctapipe
outTable[cam_id] = outfile.create_table(
"/",
cam_id,
DataTrainingOutput,
)
outData[cam_id] = outTable[cam_id].row
moments = hillas_dict[tel_id]
ellipticity = moments.width / moments.length
# Write to file also the Hillas parameters that have been used
# to calculate reco_results
moments_reco = hillas_dict_reco[tel_id]
ellipticity_reco = moments_reco.width / moments_reco.length
outData[cam_id]["good_image"] = good_for_reco[tel_id]
outData[cam_id]["is_valid"] = is_valid
outData[cam_id]["impact_dist"] = impact_dict[tel_id].to(
"m").value
outData[cam_id]["max_signal_cam"] = max_signals[tel_id]
outData[cam_id]["hillas_intensity"] = moments.intensity
outData[cam_id]["N_LST"] = n_tels["LST_LST_LSTCam"]
outData[cam_id]["N_MST"] = (n_tels["MST_MST_NectarCam"] +
n_tels["MST_MST_FlashCam"] +
n_tels["MST_SCT_SCTCam"])
outData[cam_id]["N_SST"] = (n_tels["SST_1M_DigiCam"] +
n_tels["SST_ASTRI_ASTRICam"] +
n_tels["SST_GCT_CHEC"])
outData[cam_id]["hillas_width"] = moments.width.to("deg").value
outData[cam_id]["hillas_length"] = moments.length.to(
"deg").value
outData[cam_id]["hillas_psi"] = moments.psi.to("deg").value
outData[cam_id]["hillas_skewness"] = moments.skewness
outData[cam_id]["hillas_kurtosis"] = moments.kurtosis
outData[cam_id]["h_max"] = h_max.to("m").value
outData[cam_id]["err_est_pos"] = np.nan
outData[cam_id]["err_est_dir"] = np.nan
outData[cam_id][
"true_energy"] = event.simulation.shower.energy.to(
"TeV").value
outData[cam_id]["hillas_x"] = moments.x.to("deg").value
outData[cam_id]["hillas_y"] = moments.y.to("deg").value
outData[cam_id]["hillas_phi"] = moments.phi.to("deg").value
outData[cam_id]["hillas_r"] = moments.r.to("deg").value
outData[cam_id]["pixels"] = n_pixel_dict[tel_id]
outData[cam_id]["obs_id"] = event.index.obs_id
outData[cam_id]["event_id"] = event.index.event_id
outData[cam_id]["tel_id"] = tel_id
outData[cam_id]["xi"] = xi.to("deg").value
outData[cam_id]["reco_energy"] = reco_energy
outData[cam_id]["hillas_ellipticity"] = ellipticity.value
outData[cam_id]["clusters"] = n_cluster_dict[tel_id]
outData[cam_id]["n_tel_discri"] = n_tels["GOOD images"]
outData[cam_id][
"mc_core_x"] = event.simulation.shower.core_x.to("m").value
outData[cam_id][
"mc_core_y"] = event.simulation.shower.core_y.to("m").value
outData[cam_id]["reco_core_x"] = reco_core_x.to("m").value
outData[cam_id]["reco_core_y"] = reco_core_y.to("m").value
outData[cam_id][
"mc_h_first_int"] = event.simulation.shower.h_first_int.to(
"m").value
outData[cam_id]["offset"] = offset.to("deg").value
outData[cam_id][
"mc_x_max"] = event.simulation.shower.x_max.value # g / cm2
outData[cam_id]["alt"] = reco_result.alt.to("deg").value
outData[cam_id]["az"] = reco_result.az.to("deg").value
outData[cam_id]["reco_energy_tel"] = reco_energy_tel[tel_id]
# Variables from hillas_dist_reco
outData[cam_id]["n_tel_reco"] = n_tels["GOOD images"]
outData[cam_id]["hillas_x_reco"] = moments_reco.x.to(
"deg").value
outData[cam_id]["hillas_y_reco"] = moments_reco.y.to(
"deg").value
outData[cam_id]["hillas_phi_reco"] = moments_reco.phi.to(
"deg").value
outData[cam_id][
"hillas_ellipticity_reco"] = ellipticity_reco.value
outData[cam_id]["hillas_r_reco"] = moments_reco.r.to(
"deg").value
outData[cam_id]["hillas_skewness_reco"] = moments_reco.skewness
outData[cam_id]["hillas_kurtosis_reco"] = moments_reco.kurtosis
outData[cam_id]["hillas_width_reco"] = moments_reco.width.to(
"deg").value
outData[cam_id]["hillas_length_reco"] = moments_reco.length.to(
"deg").value
outData[cam_id]["hillas_psi_reco"] = moments_reco.psi.to(
"deg").value
outData[cam_id][
"hillas_intensity_reco"] = moments_reco.intensity
outData[cam_id][
"leakage_intensity_width_1_reco"] = leakage_dict[tel_id][
"leak1_reco"]
outData[cam_id][
"leakage_intensity_width_2_reco"] = leakage_dict[tel_id][
"leak2_reco"]
outData[cam_id]["leakage_intensity_width_1"] = leakage_dict[
tel_id]["leak1"]
outData[cam_id]["leakage_intensity_width_2"] = leakage_dict[
tel_id]["leak2"]
outData[cam_id]["concentration_cog"] = concentration_dict[
tel_id]["concentration_cog"]
outData[cam_id]["concentration_core"] = concentration_dict[
tel_id]["concentration_core"]
outData[cam_id]["concentration_pixel"] = concentration_dict[
tel_id]["concentration_pixel"]
# =======================
# IMAGES INFORMATION
# =======================
if args.save_images is True:
# we define and save images content here, to make it
# adaptive to different cameras
outData[cam_id]["true_image"] = true_image[tel_id]
outData[cam_id]["reco_image"] = reco_image[tel_id]
outData[cam_id]["cleaning_mask_reco"] = cleaning_mask_reco[
tel_id]
outData[cam_id][
"cleaning_mask_clusters"] = cleaning_mask_clusters[
tel_id]
# =======================
outData[cam_id].append()
if signal_handler.stop:
break
if signal_handler.stop:
break
# make sure that all the events are properly stored
for table in outTable.values():
table.flush()
print(bcolors.BOLD +
"\n\n==================================================\n" +
"Statistical summary of processed events and images\n" +
"==================================================\n"
# + bcolors.ENDC
)
evt_cutflow()
# Catch specific cases
triggered_events = evt_cutflow.cuts["min2Tels trig"][1]
reconstructed_events = evt_cutflow.cuts["min2Tels reco"][1]
if triggered_events == 0:
print("\033[93mWARNING: No events have been triggered"
" by the selected telescopes! \033[0m")
else:
print("\n")
img_cutflow()
if reconstructed_events == 0:
print("\033[93m WARNING: None of the triggered events have been "
"properly reconstructed by the selected telescopes!\n"
"DL1 file will be empty! \033[0m")
print(bcolors.ENDC)
class Meta(tb.IsDescription):
elapsed_day = tb.Int32Col()
observed_day = tb.Int32Col() # MJD
band = tb.Int8Col()
index = tb.Int8Col()
import sys
from time import perf_counter as clock
from time import process_time as cpuclock
import numpy as np
import tables as tb
class Test(tb.IsDescription):
ngroup = tb.Int32Col(pos=1)
ntable = tb.Int32Col(pos=2)
nrow = tb.Int32Col(pos=3)
#string = StringCol(itemsize=500, pos=4)
TestDict = {
"ngroup": tb.Int32Col(pos=1),
"ntable": tb.Int32Col(pos=2),
"nrow": tb.Int32Col(pos=3),
}
def createFileArr(filename, ngroups, ntables, nrows):
# First, create the groups
# Open a file in "w"rite mode
fileh = tb.open_file(filename, mode="w", title="PyTables Stress Test")
for k in range(ngroups):
# Create the group
fileh.create_group("/", 'group%04d' % k, "Group %d" % k)
class Meta(tb.IsDescription):
observed_day = tb.Int32Col() # MJD
band = tb.Int8Col()
def _type_to_col(t, pos):
if t is int:
return tables.Int32Col(pos=pos)
if t is float:
return tables.Float32Col(pos=pos)
raise NotImplementedError(t)
class Nafc_Gap_Laser(Nafc_Gap):
PARAMS = copy(Nafc_Gap.PARAMS)
PARAMS['laser_probability'] = {
'tag':
'Probability (of trials whose targets match laser_mode) of laser being turned on (0-1)',
'type': 'float'
}
PARAMS['laser_mode'] = {
'tag': 'Laser Mode',
'type': 'list',
'values': {
'L': 0,
'R': 1,
'Both': 2
}
}
PARAMS['laser_freq'] = {
'tag': 'Laser Pulse Frequency (Hz)',
'type': 'float'
}
PARAMS['laser_duty_cycle'] = {
'tag': 'Laser Duty Cycle (0-1)',
'type': 'float'
}
PARAMS['laser_durations'] = {
'tag':
'Laser durations (ms), list-like [10, 20]. if blank, use durations from stimuli',
'type': 'str'
}
HARDWARE = copy(Nafc_Gap.HARDWARE)
HARDWARE['LASERS'] = {'L': gpio.Digital_Out, 'R': gpio.Digital_Out}
HARDWARE['LEDS']['TOP'] = gpio.Digital_Out
TrialData = copy(Nafc_Gap.TrialData)
TrialData.laser = tables.Int32Col()
TrialData.laser_duration = tables.Float32Col()
def __init__(self, laser_probability: float, laser_mode: str,
laser_freq: float, laser_duty_cycle: float,
laser_durations: typing.Union[str, list], **kwargs):
"""
Gap detection task with ability to control lasers via TTL logic for optogenetics
.. note::
Subclasses like these will be made obsolete with the completion of stimulus managers
Args:
laser_probability (float):
laser_mode:
laser_freq:
laser_duty_cycle:
laser_durations:
"""
self.laser_probability = float(laser_probability)
self.laser_mode = laser_mode
self.laser_freq = float(laser_freq)
self.laser_duty_cycle = float(laser_duty_cycle)
self.laser_durations = laser_durations
super(Nafc_Gap_Laser, self).__init__(**kwargs)
# --------------------------------------
# create description of laser pulses
# make a pair of lists, values (on/off) and durations (ms)
# use them to create pigpio scripts using the Digital_Out.store_series() method
# get the durations of on and off for a single cycle
cycle_duration = (1 / self.laser_freq) * 1000
duty_cycle_on = self.laser_duty_cycle * cycle_duration
duty_cycle_off = cycle_duration - duty_cycle_on
self.duration_ids = []
if isinstance(self.laser_durations, list):
# iterate through durations and create lists for each
for duration in self.laser_durations:
# get number of repeats to make
n_cycles = np.floor(duration / cycle_duration)
durations = [duty_cycle_on, duty_cycle_off] * n_cycles
values = [1, 0] * n_cycles
# pad any incomplete cycles
dur_remaining = duration - (cycle_duration * n_cycles)
if dur_remaining < duty_cycle_on:
durations.append(dur_remaining)
values.append(1)
else:
durations.extend(
[duty_cycle_on, dur_remaining - duty_cycle_on])
values.extend([1, 0])
# store pulses as pigpio scripts
self.hardware['LASERS']['L'].store_series(duration,
values=values,
durations=durations)
self.hardware['LASERS']['R'].store_series(duration,
values=values,
durations=durations)
else:
# use the durations of the stimuli
self.logger.exception(
'reading durations from stimulus manager not implemented')
raise NotImplementedError(
'read durations from the stimulus manager')
# -----------------------------------
# create a pulse for the LED that's equal to the longest stimulus duration
# use find_recursive to find all durations
# FIXME: implement stimulus managers properly, including API to get attributes of stimuli
stim_durations = list(find_recursive('duration', kwargs['stim']))
max_duration = np.max(stim_durations)
self.hardware['LEDS']['TOP'].store_series('on',
values=1,
durations=max_duration)
def request(self, *args, **kwargs):
# call the super method
data = super(Nafc_Gap_Laser, self).request(*args, **kwargs)
# handle laser logic
# if the laser_mode is fulfilled, roll for a laser
test_laser = False
if self.laser_mode == "L" and self.target == "L":
test_laser = True
elif self.laser_mode == "R" and self.target == "R":
test_laser = True
elif self.laser_mode == "Both":
test_laser = True
duration = 0
do_laser = False
if test_laser:
# if we've rolled correctly for a laser...
if np.random.rand() <= self.laser_probability:
do_laser = True
# pick a random duration
duration = np.random.choice(self.laser_durations)
# insert the laser triggers before the rest of the triggers
self.triggers['C'].insert(
0,
lambda: self.hardware['LASERS']['L'].series(id=duration))
self.triggers['C'].insert(
0,
lambda: self.hardware['LASERS']['R'].series(id=duration))
# always turn the light on
self.triggers['C'].insert(
0, lambda: self.hardware['LEDS']['TOP'].series(id='on'))
# store the data about the laser status
data['laser'] = do_laser
data['laser_duration'] = duration
# return the data created by the original task
return data
class Particle(tb.IsDescription):
name = tb.StringCol(16, pos=1) # 16-character String
lati = tb.Int32Col(pos=2) # integer
longi = tb.Int32Col(pos=3) # integer
vector = tb.Int32Col(shape=(2,), pos=4) # Integer
matrix2D = tb.Float64Col(shape=(2, 2), pos=5) # double (double-precision)
class DescriptionForTradesTable(tables.IsDescription):
price = tables.Float32Col()
amount = tables.Float32Col()
timestamp = tables.Float32Col()
tid = tables.Int32Col()
type = tables.StringCol(3)
