def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def test_hit_histograming(self):
raw_data = np.array([67307647, 67645759, 67660079, 67541711, 67718111, 67913663, 67914223, 67847647, 67978655, 68081199, 68219119, 68219487, 68425615, 68311343, 68490719, 68373295, 68553519, 68693039, 68573503, 68709951, 68717058, 68734735, 68604719, 68753999, 68761151, 68847327, 69014799, 69079791, 69211359, 69221055, 69279567, 69499247, 69773183, 69788527, 69998559, 69868559, 69872655, 70003599, 69902527, 70274575, 70321471, 70429983, 70563295, 70574959, 70447631, 70584591, 70783023, 71091999, 70972687, 70985087, 71214815, 71382623, 71609135, 71643519, 71720527, 71897695, 72167199, 72040047, 72264927, 72423983, 77471983, 77602863, 77604383, 77485295, 77616415, 77618927, 77619231, 77639983, 77655871, 77544159, 77548303, 77338399, 77345567, 77346287, 77360399, 77255407, 77386211, 77268287, 77279215, 77409599, 77075983, 76951903, 76980527, 77117023, 76991055, 77011007, 77148127, 77148815, 76827167, 76700031, 76868895, 76758575, 76889567, 76558303, 76429599, 76584783, 76468191, 76610943, 76613743, 76620879, 76629375, 76285999, 76321908, 76194319, 76205599, 76233759, 76065391, 76075839, 76093759, 75801311, 75826319, 75829215, 75699231, 75403887, 75565039, 75439135, 75111711, 75115151, 75251487, 75258399, 75138015, 75303471, 74974111, 74868559, 75030047, 75050079, 74714591, 74722847, 74595103, 74649935, 74656815, 74796511, 74455519, 74391519, 74402607, 74534383, 74189695, 74064911, 74246271, 74116063, 74248719, 74133119, 73935183, 73941087, 73811295, 73663583, 73743423, 73449647, 73453391, 73323743, 73343471, 73474159, 73345087, 73206751, 72899295, 72958559, 72828447, 72542623, 82383232, 67374687, 67503967, 67766575, 68179999, 68052847, 68198239, 68104495, 68235759, 68238223, 68472415, 68490463, 68501279, 68621071, 68623903, 68821791, 68988639, 68864047, 69003183, 68876015, 69007423, 68891407, 69267743, 69272367, 69159567, 69666911, 69684447, 70003247, 70018895, 69898927, 69938543, 69942031, 70198863, 70339919, 70587455, 70462783, 70597679, 70796399, 70800015, 70703887, 71121183, 71323151, 71243535, 71578703, 71467695, 71622879, 71629359, 71831264, 71836511, 71710319, 71992943, 72353855, 72355039, 77606628, 77608287, 77622047, 77510223, 77653263, 77664319, 77546223, 77677471, 77549375, 77213519, 77219551, 77232207, 77234991, 77366511, 77373791, 77389647, 77404383, 77070655, 77087199, 76956975, 76996431, 77009183, 77015327, 76683567, 76840351, 76862255, 76888804, 76548975, 76554767, 76427087, 76560159, 76451967, 76456847, 76468015, 76627295, 76352831, 76354863, 76365887, 75923999, 76074175, 75955439, 76086063, 75774239, 75781535, 75792671, 75662111, 75793647, 75797167, 75827023, 75696543, 75390527, 75522031, 75533663, 75541775, 75432255, 75571535, 75115535, 75247999, 75145197, 75151391, 75160799, 74974991, 74852831, 74871839, 74882783, 75023199, 74896943, 75028767, 75046431, 74922463, 74725711, 74621199, 74658623, 74663183, 74336383, 74484559, 74364526, 74370287, 74370639, 74517983, 74393615, 74205471, 74217359, 74227263, 74231727, 74102559, 74237999, 74248735, 73953599, 73868591, 74000703, 74002975, 73877295, 73664910, 73695967, 73704751, 73579583, 73582639, 73719055, 73405998, 73448207, 73481951, 73008831, 73175087, 73044495, 73058863, 73194895, 73197919, 73093151, 72895567, 72918543, 72947039, 72957919, 82383481, 67392015, 67303135, 67312799, 67318303, 67453727, 67454767, 67634719, 67645887, 67717391, 67914111, 67947919, 67818463, 68052959, 68097215, 68500543, 68711909, 68584735, 68726975, 68741679, 68615471, 68750559, 68755487, 68629311, 68764687, 68765648, 68990175, 69022959, 69023727, 69217327, 69547327, 69665839, 69809983, 69814815, 70006831, 70037807, 70055951, 70068511, 70184031, 70323999, 70334687, 70566095, 70588751, 70723935, 71049695, 70952031, 71084831, 71376863, 71256287, 71611039, 71487727, 71618591, 71623999, 71514239, 71891231, 71897327, 71897663, 72036783, 72391487, 77604975, 77608163, 77621327, 77501983, 77635039, 77646559, 77654671, 77655695, 77546543, 77678383, 77345471, 77224735, 77375519, 77385519, 77393967, 76944399, 76975663, 77114628, 77115231, 77127525, 77142959, 76677423, 76699967, 76722287, 76857647, 76739039, 76883567, 76891615, 76453343, 76584335, 76590623, 76594607, 76600031, 76611167, 76617743, 76622303, 76285999, 76329231, 76335839, 76348175, 76350351, 76356783, 75910383, 75639343, 75787615, 75660079, 75796895, 75797615, 75692559, 75827999, 75833487, 75836479, 75518943, 75568143, 75278943, 75290271, 75297903, 75309391, 75312479, 75315119, 74852223, 74987055, 74858047, 74992943, 74875439, 75008031, 74885407, 75027743, 75055583, 74927839, 74738719, 74629087, 74767391, 74779295, 74789343, 74791247, 74323183, 74454239, 74349455, 74364751, 74516047, 74528559, 74192207, 74201535, 74084367, 74220511, 74109039, 74263263, 74133215, 73807119, 73945313, 73868148, 74001631, 73536815, 73684815, 73711439, 73275407, 73408799, 73052767, 73190975, 73209823, 72788271, 72960607, 72487647, 82383730, 67407151, 67415583, 67322127, 67523871, 67700959, 67583039, 67905375, 67793199, 68159583, 68237791, 68306479, 68492399], np.uint32)
interpreter = PyDataInterpreter()
histograming = PyDataHistograming()
interpreter.set_trig_count(1)
interpreter.set_warning_output(False)
histograming.set_no_scan_parameter()
histograming.create_occupancy_hist(True)
interpreter.interpret_raw_data(raw_data)
interpreter.store_event()
histograming.add_hits(interpreter.get_hits())
occ_hist_cpp = histograming.get_occupancy()[:, :, 0]
col_arr, row_arr = convert_data_array(raw_data, filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
occ_hist_python, _, _ = np.histogram2d(col_arr, row_arr, bins=(80, 336), range=[[1, 80], [1, 336]])
self.assertTrue(np.all(occ_hist_cpp == occ_hist_python))
def configure(self):
if self.trig_count == 0:
self.consecutive_lvl1 = (2 ** self.register.global_registers['Trig_Count']['bitlength'])
else:
self.consecutive_lvl1 = self.trig_count
if self.occupancy_limit * self.n_triggers * self.consecutive_lvl1 < 1.0:
logging.warning('Number of triggers too low for given occupancy limit. Any noise hit will lead to a masked pixel.')
commands = []
commands.extend(self.register.get_commands("ConfMode"))
# TDAC
tdac_max = 2 ** self.register.pixel_registers['TDAC']['bitlength'] - 1
self.register.set_pixel_register_value("TDAC", tdac_max)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span)
# Enable
if self.use_enable_mask:
self.register.set_pixel_register_value("Enable", np.logical_and(mask, self.register.get_pixel_register_value("Enable")))
else:
self.register.set_pixel_register_value("Enable", mask)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="Enable"))
# Imon
self.register.set_pixel_register_value('Imon', 1)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='Imon'))
# C_High
self.register.set_pixel_register_value('C_High', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
# C_Low
self.register.set_pixel_register_value('C_Low', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
# Registers
# self.register.set_global_register_value("Trig_Lat", self.trigger_latency) # set trigger latency
self.register.set_global_register_value("Trig_Count", self.trig_count) # set number of consecutive triggers
commands.extend(self.register.get_commands("WrRegister", name=["Trig_Count"]))
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_trig_count(self.trig_count)
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
def setUpClass(cls):
cls.interpreter = PyDataInterpreter()
cls.histogram = PyDataHistograming()
cls.clusterizer = PyDataClusterizer()
with AnalyzeRawData(raw_data_file=tests_data_folder + 'unit_test_data_1.h5', analyzed_data_file=tests_data_folder + 'unit_test_data_1_interpreted.h5', create_pdf=False) as analyze_raw_data: # analyze the digital scan raw data, do not show any feedback (no prints to console, no plots)
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_hit_table = True # can be set to false to omit hit table creation, std. setting is false
analyze_raw_data.create_cluster_hit_table = True # adds the cluster id and seed info to each hit, std. setting is false
analyze_raw_data.create_cluster_table = True # enables the creation of a table with all clusters, std. setting is false
analyze_raw_data.create_trigger_error_hist = True # creates a histogram summing up the trigger errors
analyze_raw_data.create_cluster_size_hist = True # enables cluster size histogramming, can save some time, std. setting is false
analyze_raw_data.create_cluster_tot_hist = True # enables cluster ToT histogramming per cluster size, std. setting is false
analyze_raw_data.create_meta_word_index = True # stores the start and stop raw data word index for every event, std. setting is false
analyze_raw_data.create_meta_event_index = True # stores the event number for each readout in an additional meta data array, default: False
analyze_raw_data.interpret_word_table(use_settings_from_file=False, fei4b=False) # the actual start conversion command
with AnalyzeRawData(raw_data_file=tests_data_folder + 'unit_test_data_2.h5', analyzed_data_file=tests_data_folder + 'unit_test_data_2_interpreted.h5', create_pdf=False) as analyze_raw_data: # analyze the fast threshold scan raw data, do not show any feedback (no prints to console, no plots)
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_threshold_hists = True # makes only sense if threshold scan data is analyzed, std. setting is false
analyze_raw_data.interpret_word_table(use_settings_from_file=False, fei4b=False) # the actual start conversion command
with AnalyzeRawData(raw_data_file=None, analyzed_data_file=tests_data_folder + 'unit_test_data_1_interpreted.h5', create_pdf=False) as analyze_raw_data: # analyze the digital scan hit data, do not show any feedback (no prints to console, no plots)
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_cluster_hit_table = True
analyze_raw_data.create_cluster_table = True
analyze_raw_data.create_cluster_size_hist = True
analyze_raw_data.create_cluster_tot_hist = True
analyze_raw_data.analyze_hit_table(analyzed_data_out_file=tests_data_folder + 'unit_test_data_1_analyzed.h5')
with AnalyzeRawData(raw_data_file=tests_data_folder + 'unit_test_data_3.h5', analyzed_data_file=tests_data_folder + 'unit_test_data_3_interpreted.h5', create_pdf=False) as analyze_raw_data: # analyze the digital scan raw data per scan parameter, do not show any feedback (no prints to console, no plots)
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_hit_table = True # can be set to false to omit hit table creation, std. setting is false
analyze_raw_data.create_cluster_hit_table = True # adds the cluster id and seed info to each hit, std. setting is false
analyze_raw_data.create_cluster_table = True # enables the creation of a table with all clusters, std. setting is false
analyze_raw_data.create_trigger_error_hist = True # creates a histogram summing up the trigger errors
analyze_raw_data.create_cluster_size_hist = True # enables cluster size histogramming, can save some time, std. setting is false
analyze_raw_data.create_cluster_tot_hist = True # enables cluster ToT histogramming per cluster size, std. setting is false
analyze_raw_data.create_meta_word_index = True # stores the start and stop raw data word index for every event, std. setting is false
analyze_raw_data.create_meta_event_index = True # stores the event number for each readout in an additional meta data array, default: False
analyze_raw_data.interpret_word_table(use_settings_from_file=False, fei4b=False) # the actual start conversion command
with AnalyzeRawData(raw_data_file=tests_data_folder + 'unit_test_data_2.h5', analyzed_data_file=tests_data_folder + 'unit_test_data_2_hits.h5', create_pdf=False) as analyze_raw_data: # analyze the fast threshold scan raw data, do not show any feedback (no prints to console, no plots)
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_hit_table = True
analyze_raw_data.create_threshold_hists = True # makes only sense if threshold scan data is analyzed, std. setting is false
analyze_raw_data.interpret_word_table(use_settings_from_file=False, fei4b=False) # the actual start conversion command
with AnalyzeRawData(raw_data_file=None, analyzed_data_file=tests_data_folder + 'unit_test_data_2_hits.h5', create_pdf=False) as analyze_raw_data:
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_threshold_hists = True
analyze_raw_data.analyze_hit_table(analyzed_data_out_file=tests_data_folder + 'unit_test_data_2_analyzed.h5')
with AnalyzeRawData(raw_data_file=tests_data_folder + 'unit_test_data_4.h5', analyzed_data_file=tests_data_folder + 'unit_test_data_4_interpreted.h5', create_pdf=False) as analyze_raw_data:
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_hit_table = True
analyze_raw_data.interpret_word_table(use_settings_from_file=False, fei4b=False) # the actual start conversion command
with AnalyzeRawData(raw_data_file=[tests_data_folder + 'unit_test_data_4_parameter_128.h5', tests_data_folder + 'unit_test_data_4_parameter_256.h5'], analyzed_data_file=tests_data_folder + 'unit_test_data_4_interpreted_2.h5', scan_parameter_name='parameter', create_pdf=False) as analyze_raw_data:
analyze_raw_data.chunk_size = 2999999
analyze_raw_data.create_hit_table = True
analyze_raw_data.interpret_word_table(use_settings_from_file=False, fei4b=False) # the actual start conversion command
def test_libraries_stability(self): # calls 50 times the constructor and destructor to check the libraries
progress_bar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.ETA()], maxval=50, term_width=80)
progress_bar.start()
for i in range(50):
interpreter = PyDataInterpreter()
histogram = PyDataHistograming()
clusterizer = PyDataClusterizer()
del interpreter
del histogram
del clusterizer
progress_bar.update(i)
progress_bar.finish()
def configure(self):
if self.trig_count == 0:
self.consecutive_lvl1 = (2 ** self.register.global_registers['Trig_Count']['bitlength'])
else:
self.consecutive_lvl1 = self.trig_count
self.abs_occ_limit = int(self.occupancy_limit * self.n_triggers * self.consecutive_lvl1)
if self.abs_occ_limit <= 0:
logging.info('Any noise hit will lead to an increased pixel threshold.')
else:
logging.info('The pixel threshold of any pixel with an occpancy >%d will be increased' % self.abs_occ_limit)
commands = []
commands.extend(self.register.get_commands("ConfMode"))
# TDAC
tdac_max = 2 ** self.register.pixel_registers['TDAC']['bitlength'] - 1
self.register.set_pixel_register_value("TDAC", tdac_max)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span)
# Enable
if self.use_enable_mask:
self.register.set_pixel_register_value("Enable", np.logical_and(mask, self.register.get_pixel_register_value("Enable")))
else:
self.register.set_pixel_register_value("Enable", mask)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="Enable"))
# Imon
self.register.set_pixel_register_value('Imon', 1)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='Imon'))
# C_High
self.register.set_pixel_register_value('C_High', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
# C_Low
self.register.set_pixel_register_value('C_Low', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
# Registers
# self.register.set_global_register_value("Trig_Lat", self.trigger_latency) # set trigger latency
self.register.set_global_register_value("Trig_Count", self.trig_count) # set number of consecutive triggers
commands.extend(self.register.get_commands("WrRegister", name=["Trig_Count"]))
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_trig_count(self.trig_count)
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
class DataWorker(QtCore.QObject):
run_start = QtCore.pyqtSignal()
config_data = QtCore.pyqtSignal(dict)
interpreted_data = QtCore.pyqtSignal(dict)
meta_data = QtCore.pyqtSignal(dict)
finished = QtCore.pyqtSignal()
def __init__(self):
QtCore.QObject.__init__(self)
self.integrate_readouts = 1
self.n_readout = 0
self._stop_readout = Event()
self.setup_raw_data_analysis()
def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def connect(self, socket_addr):
self.socket_addr = socket_addr
self.context = zmq.Context()
self.socket_pull = self.context.socket(zmq.PULL)
self.socket_pull.connect(self.socket_addr)
self.poller = zmq.Poller() # poll needed to be able to return QThread
self.poller.register(self.socket_pull, zmq.POLLIN)
@pyqtSlot(float)
def on_set_integrate_readouts(self, value):
self.integrate_readouts = value
def analyze_raw_data(self, raw_data):
self.interpreter.interpret_raw_data(raw_data)
self.histograming.add_hits(self.interpreter.get_hits())
@pyqtSlot()
def process_data(self): # infinite loop via QObject.moveToThread(), does not block event loop
while(not self._stop_readout.wait(0.001)): # use wait(), do not block here
# socks = dict(self.poller.poll(1))
# if self.socket_pull in socks and socks[self.socket_pull] == zmq.POLLIN:
# meta_data = self.socket_pull.recv_json()
try:
meta_data = self.socket_pull.recv_json(flags=zmq.NOBLOCK)
except zmq.Again:
pass
else:
name = meta_data.pop('name')
if name == 'FEI4readoutData':
data = self.socket_pull.recv()
# reconstruct numpy array
buf = buffer(data)
dtype = meta_data.pop('dtype')
shape = meta_data.pop('shape')
data_array = np.frombuffer(buf, dtype=dtype).reshape(shape)
# count readouts and reset
self.n_readout += 1
if self.integrate_readouts != 0 and self.n_readout % self.integrate_readouts == 0:
self.histograming.reset()
# we do not want to reset interpreter to keep the error counters
# self.interpreter.reset()
# interpreted data
self.analyze_raw_data(data_array)
if self.integrate_readouts == 0 or self.n_readout % self.integrate_readouts == self.integrate_readouts - 1:
interpreted_data = {
'occupancy': self.histograming.get_occupancy(),
'tot_hist': self.histograming.get_tot_hist(),
'tdc_counters': self.interpreter.get_tdc_counters(),
'error_counters': self.interpreter.get_error_counters(),
'service_records_counters': self.interpreter.get_service_records_counters(),
'trigger_error_counters': self.interpreter.get_trigger_error_counters(),
'rel_bcid_hist': self.histograming.get_rel_bcid_hist()}
self.interpreted_data.emit(interpreted_data)
# meta data
meta_data.update({'n_hits': self.interpreter.get_n_hits(), 'n_events': self.interpreter.get_n_events()})
self.meta_data.emit(meta_data)
elif name == 'RunConf':
# TODO: from FE config
try:
n_bcid = int(data[1]['trig_count'])
except KeyError:
n_bcid = 0
self.histograming.reset()
self.interpreter.reset()
self.interpreter.set_trig_count(n_bcid)
self.run_start.emit()
self.config_data.emit(meta_data)
self.finished.emit()
# @pyqtSlot()
def stop(self):
self._stop_readout.set()
class DataWorker(QtCore.QObject):
run_start = QtCore.pyqtSignal()
config_data = QtCore.pyqtSignal(dict)
interpreted_data = QtCore.pyqtSignal(dict)
meta_data = QtCore.pyqtSignal(dict)
finished = QtCore.pyqtSignal()
def __init__(self):
QtCore.QObject.__init__(self)
self.integrate_readouts = 1
self.n_readout = 0
self._stop_readout = Event()
self.setup_raw_data_analysis()
def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def connect(self, socket_addr):
self.socket_addr = socket_addr
self.context = zmq.Context()
self.socket_pull = self.context.socket(zmq.PULL)
self.socket_pull.connect(self.socket_addr)
self.poller = zmq.Poller() # poll needed to be able to return QThread
self.poller.register(self.socket_pull, zmq.POLLIN)
@pyqtSlot(float)
def on_set_integrate_readouts(self, value):
self.integrate_readouts = value
def analyze_raw_data(self, raw_data):
self.interpreter.interpret_raw_data(raw_data)
self.histograming.add_hits(self.interpreter.get_hits())
@pyqtSlot()
def process_data(
self
): # infinite loop via QObject.moveToThread(), does not block event loop
while (not self._stop_readout.wait(0.001)
): # use wait(), do not block here
# socks = dict(self.poller.poll(1))
# if self.socket_pull in socks and socks[self.socket_pull] == zmq.POLLIN:
# meta_data = self.socket_pull.recv_json()
try:
meta_data = self.socket_pull.recv_json(flags=zmq.NOBLOCK)
except zmq.Again:
pass
else:
name = meta_data.pop('name')
if name == 'FEI4readoutData':
data = self.socket_pull.recv()
# reconstruct numpy array
buf = buffer(data)
dtype = meta_data.pop('dtype')
shape = meta_data.pop('shape')
data_array = np.frombuffer(buf, dtype=dtype).reshape(shape)
# count readouts and reset
self.n_readout += 1
if self.integrate_readouts != 0 and self.n_readout % self.integrate_readouts == 0:
self.histograming.reset()
# we do not want to reset interpreter to keep the error counters
# self.interpreter.reset()
# interpreted data
self.analyze_raw_data(data_array)
if self.integrate_readouts == 0 or self.n_readout % self.integrate_readouts == self.integrate_readouts - 1:
interpreted_data = {
'occupancy':
self.histograming.get_occupancy(),
'tot_hist':
self.histograming.get_tot_hist(),
'tdc_counters':
self.interpreter.get_tdc_counters(),
'error_counters':
self.interpreter.get_error_counters(),
'service_records_counters':
self.interpreter.get_service_records_counters(),
'trigger_error_counters':
self.interpreter.get_trigger_error_counters(),
'rel_bcid_hist':
self.histograming.get_rel_bcid_hist()
}
self.interpreted_data.emit(interpreted_data)
# meta data
meta_data.update({
'n_hits':
self.interpreter.get_n_hits(),
'n_events':
self.interpreter.get_n_events()
})
self.meta_data.emit(meta_data)
elif name == 'RunConf':
# TODO: from FE config
try:
n_bcid = int(data[1]['trig_count'])
except KeyError:
n_bcid = 0
self.histograming.reset()
self.interpreter.reset()
self.interpreter.set_trig_count(n_bcid)
self.run_start.emit()
self.config_data.emit(meta_data)
self.finished.emit()
# @pyqtSlot()
def stop(self):
self._stop_readout.set()
class ThresholdBaselineTuning(Fei4RunBase):
'''Threshold Baseline Tuning
Tuning the FEI4 to the lowest possible threshold (GDAC and TDAC). Feedback current will not be tuned.
NOTE: In case of RX errors decrease the trigger frequency (= increase trigger_rate_limit)
'''
_default_run_conf = {
"occupancy_limit": 0, # occupancy limit, when reached the TDAC will be decreased (increasing threshold). 0 will mask any pixel with occupancy greater than zero
"scan_parameters": [('Vthin_AltFine', (120, None)), ('Step', 60)], # the Vthin_AltFine range, number of steps (repetition at constant Vthin_AltFine)
"increase_threshold": 5, # increase the threshold in VthinAF after tuning
"disabled_pixels_limit": 0.01, # limit of disabled pixels, fraction of all pixels
"use_enable_mask": False, # if True, enable mask from config file anded with mask (from col_span and row_span), if False use mask only for enable mask
"n_triggers": 10000, # total number of trigger sent to FE
"trigger_rate_limit": 500, # artificially limiting the trigger rate, in BCs (25ns)
"trig_count": 15, # FE global register Trig_Count
"col_span": [1, 80], # column range (from minimum to maximum value). From 1 to 80.
"row_span": [1, 336], # row range (from minimum to maximum value). From 1 to 336.
}
def configure(self):
if self.trig_count == 0:
self.consecutive_lvl1 = (2 ** self.register.global_registers['Trig_Count']['bitlength'])
else:
self.consecutive_lvl1 = self.trig_count
if self.occupancy_limit * self.n_triggers * self.consecutive_lvl1 < 1.0:
logging.warning('Number of triggers too low for given occupancy limit. Any noise hit will lead to a masked pixel.')
commands = []
commands.extend(self.register.get_commands("ConfMode"))
# TDAC
tdac_max = 2 ** self.register.pixel_registers['TDAC']['bitlength'] - 1
self.register.set_pixel_register_value("TDAC", tdac_max)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span)
# Enable
if self.use_enable_mask:
self.register.set_pixel_register_value("Enable", np.logical_and(mask, self.register.get_pixel_register_value("Enable")))
else:
self.register.set_pixel_register_value("Enable", mask)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="Enable"))
# Imon
self.register.set_pixel_register_value('Imon', 1)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='Imon'))
# C_High
self.register.set_pixel_register_value('C_High', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
# C_Low
self.register.set_pixel_register_value('C_Low', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
# Registers
# self.register.set_global_register_value("Trig_Lat", self.trigger_latency) # set trigger latency
self.register.set_global_register_value("Trig_Count", self.trig_count) # set number of consecutive triggers
commands.extend(self.register.get_commands("WrRegister", name=["Trig_Count"]))
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_trig_count(self.trig_count)
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
def scan(self):
scan_parameter_range = [self.register.get_global_register_value("Vthin_AltFine"), 0]
if self.scan_parameters.Vthin_AltFine[0]:
scan_parameter_range[0] = self.scan_parameters.Vthin_AltFine[0]
if self.scan_parameters.Vthin_AltFine[1]:
scan_parameter_range[1] = self.scan_parameters.Vthin_AltFine[1]
steps = 1
if self.scan_parameters.Step:
steps = self.scan_parameters.Step
lvl1_command = self.register.get_commands("LV1")[0] + self.register.get_commands("zeros", length=self.trigger_rate_limit)[0]
self.total_scan_time = int(lvl1_command.length() * 25 * (10 ** -9) * self.n_triggers)
disabled_pixels_limit_cnt = int(self.disabled_pixels_limit * 336 * 80)
preselected_pixels = invert_pixel_mask(self.register.get_pixel_register_value('Enable')).sum()
disabled_pixels = 0
for reg_val in range(scan_parameter_range[0], scan_parameter_range[1] - 1, -1):
if self.stop_run.is_set():
break
self.register.create_restore_point(name=str(reg_val))
logging.info('Scanning Vthin_AltFine %d' % reg_val)
commands = []
commands.extend(self.register.get_commands("ConfMode"))
self.register.set_global_register_value("Vthin_AltFine", reg_val) # set number of consecutive triggers
commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltFine"]))
# setting FE into RunMode
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
step = 0
while True:
if self.stop_run.is_set():
break
self.histograming.reset()
step += 1
logging.info('Step %d / %d at Vthin_AltFine %d' % (step, steps, reg_val))
logging.info('Estimated scan time: %ds' % self.total_scan_time)
with self.readout(Vthin_AltFine=reg_val, Step=step, reset_sram_fifo=True, fill_buffer=True, clear_buffer=True, callback=self.handle_data):
got_data = False
start = time()
self.register_utils.send_command(lvl1_command, repeat=self.n_triggers, wait_for_finish=False, set_length=True, clear_memory=False)
while not self.stop_run.wait(0.1):
if self.register_utils.is_ready:
if got_data:
self.progressbar.finish()
logging.info('Finished sending %d triggers' % self.n_triggers)
break
if not got_data:
if self.fifo_readout.data_words_per_second() > 0:
got_data = True
logging.info('Taking data...')
self.progressbar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.Timer()], maxval=self.total_scan_time, poll=10, term_width=80).start()
else:
try:
self.progressbar.update(time() - start)
except ValueError:
pass
# Use fast C++ hit histograming to save time
raw_data = np.ascontiguousarray(data_array_from_data_iterable(self.fifo_readout.data), dtype=np.uint32)
self.interpreter.interpret_raw_data(raw_data)
self.interpreter.store_event() # force to create latest event
self.histograming.add_hits(self.interpreter.get_hits())
occ_hist = self.histograming.get_occupancy()[:, :, 0]
# noisy pixels are set to 1
occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
occ_mask[occ_hist > self.occupancy_limit * self.n_triggers * self.consecutive_lvl1] = 1
tdac_reg = self.register.get_pixel_register_value('TDAC')
decrease_pixel_mask = np.logical_and(occ_mask > 0, tdac_reg > 0)
disable_pixel_mask = np.logical_and(occ_mask > 0, tdac_reg == 0)
enable_reg = self.register.get_pixel_register_value('Enable')
enable_mask = np.logical_and(enable_reg, invert_pixel_mask(disable_pixel_mask))
if np.logical_and(occ_mask > 0, enable_reg == 0).sum():
logging.warning('Received data from disabled pixels')
# disabled_pixels += disable_pixel_mask.sum() # can lead to wrong values if the enable reg is corrupted
disabled_pixels = invert_pixel_mask(enable_mask).sum() - preselected_pixels
if disabled_pixels > disabled_pixels_limit_cnt:
logging.info('Limit of disabled pixels reached: %d (limit %d)... stopping scan' % (disabled_pixels, disabled_pixels_limit_cnt))
self.register.restore(name=str(reg_val))
break
else:
logging.info('Increasing threshold of %d pixel(s)' % (decrease_pixel_mask.sum(),))
logging.info('Disabling %d pixel(s), total number of disabled pixel(s): %d' % (disable_pixel_mask.sum(), disabled_pixels))
tdac_reg[decrease_pixel_mask] -= 1
self.register.set_pixel_register_value('TDAC', tdac_reg)
self.register.set_pixel_register_value('Enable', enable_mask)
commands = []
commands.extend(self.register.get_commands("ConfMode"))
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='TDAC'))
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Enable'))
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
if occ_mask.sum() == 0 or step == steps or decrease_pixel_mask.sum() < disabled_pixels_limit_cnt:
self.register.clear_restore_points(name=str(reg_val))
self.last_tdac_distribution = self.register.get_pixel_register_value('TDAC')
self.last_occupancy_hist = occ_hist.copy()
self.last_occupancy_mask = occ_mask.copy()
self.last_reg_val = reg_val
self.last_step = step
break
else:
logging.info('Found %d noisy pixels... repeat tuning step for Vthin_AltFine %d' % (occ_mask.sum(), reg_val))
if disabled_pixels > disabled_pixels_limit_cnt:
self.last_good_threshold = self.register.get_global_register_value("Vthin_AltFine")
self.last_good_tdac = self.register.get_pixel_register_value('TDAC')
self.last_good_enable_mask = self.register.get_pixel_register_value('Enable')
break
def analyze(self):
self.register.set_global_register_value("Vthin_AltFine", self.last_good_threshold + self.increase_threshold)
self.register.set_pixel_register_value('TDAC', self.last_good_tdac)
self.register.set_pixel_register_value('Enable', self.last_good_enable_mask)
with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
analyze_raw_data.create_source_scan_hist = True
analyze_raw_data.interpreter.set_warning_output(False)
analyze_raw_data.interpret_word_table()
analyze_raw_data.interpreter.print_summary()
analyze_raw_data.plot_histograms()
plot_occupancy(self.last_occupancy_hist.T, title='Noisy Pixels at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), filename=analyze_raw_data.output_pdf)
plot_fancy_occupancy(self.last_occupancy_hist.T, filename=analyze_raw_data.output_pdf)
plot_occupancy(self.last_occupancy_mask.T, title='Occupancy Mask at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), z_max=1, filename=analyze_raw_data.output_pdf)
plot_fancy_occupancy(self.last_occupancy_mask.T, filename=analyze_raw_data.output_pdf)
plotThreeWay(self.last_tdac_distribution.T, title='TDAC at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), x_axis_title="TDAC", filename=analyze_raw_data.output_pdf, maximum=31, bins=32)
plot_occupancy(self.last_tdac_distribution.T, title='TDAC at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), z_max=31, filename=analyze_raw_data.output_pdf)
plot_occupancy(self.register.get_pixel_register_value('Enable').T, title='Enable Mask', z_max=1, filename=analyze_raw_data.output_pdf)
plot_fancy_occupancy(self.register.get_pixel_register_value('Enable').T, filename=analyze_raw_data.output_pdf)
class ThresholdBaselineTuning(Fei4RunBase):
'''Threshold Baseline Tuning
Tuning the FEI4 to the lowest possible threshold (GDAC and TDAC). Feedback current will not be tuned.
NOTE: In case of RX errors decrease the trigger frequency (= increase trigger_rate_limit)
'''
_default_run_conf = {
"occupancy_limit": 0, # occupancy limit, when reached the TDAC will be decreased (increasing threshold). 0 will mask any pixel with occupancy greater than zero
"scan_parameters": [('Vthin_AltFine', (120, None)), ('Step', 60)], # the Vthin_AltFine range, number of steps (repetition at constant Vthin_AltFine)
"increase_threshold": 5, # increase the threshold in VthinAF after tuning
"disabled_pixels_limit": 0.01, # limit of disabled pixels, fraction of all pixels
"use_enable_mask": False, # if True, enable mask from config file anded with mask (from col_span and row_span), if False use mask only for enable mask
"n_triggers": 10000, # total number of trigger sent to FE
"trigger_rate_limit": 500, # artificially limiting the trigger rate, in BCs (25ns)
"trig_count": 0, # FE-I4 trigger count, number of consecutive BCs, 0 means 16, from 0 to 15
"col_span": [1, 80], # column range (from minimum to maximum value). From 1 to 80.
"row_span": [1, 336], # row range (from minimum to maximum value). From 1 to 336.
}
def configure(self):
if self.trig_count == 0:
self.consecutive_lvl1 = (2 ** self.register.global_registers['Trig_Count']['bitlength'])
else:
self.consecutive_lvl1 = self.trig_count
if self.occupancy_limit * self.n_triggers * self.consecutive_lvl1 < 1.0:
logging.warning('Number of triggers too low for given occupancy limit. Any noise hit will lead to a masked pixel.')
commands = []
commands.extend(self.register.get_commands("ConfMode"))
# TDAC
tdac_max = 2 ** self.register.pixel_registers['TDAC']['bitlength'] - 1
self.register.set_pixel_register_value("TDAC", tdac_max)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="TDAC"))
mask = make_box_pixel_mask_from_col_row(column=self.col_span, row=self.row_span)
# Enable
if self.use_enable_mask:
self.register.set_pixel_register_value("Enable", np.logical_and(mask, self.register.get_pixel_register_value("Enable")))
else:
self.register.set_pixel_register_value("Enable", mask)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name="Enable"))
# Imon
self.register.set_pixel_register_value('Imon', 1)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='Imon'))
# C_High
self.register.set_pixel_register_value('C_High', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_High'))
# C_Low
self.register.set_pixel_register_value('C_Low', 0)
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=True, name='C_Low'))
# Registers
# self.register.set_global_register_value("Trig_Lat", self.trigger_latency) # set trigger latency
self.register.set_global_register_value("Trig_Count", self.trig_count) # set number of consecutive triggers
commands.extend(self.register.get_commands("WrRegister", name=["Trig_Count"]))
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_trig_count(self.trig_count)
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
def scan(self):
scan_parameter_range = [self.register.get_global_register_value("Vthin_AltFine"), 0]
if self.scan_parameters.Vthin_AltFine[0]:
scan_parameter_range[0] = self.scan_parameters.Vthin_AltFine[0]
if self.scan_parameters.Vthin_AltFine[1]:
scan_parameter_range[1] = self.scan_parameters.Vthin_AltFine[1]
steps = 1
if self.scan_parameters.Step:
steps = self.scan_parameters.Step
lvl1_command = self.register.get_commands("LV1")[0] + self.register.get_commands("zeros", length=self.trigger_rate_limit)[0]
self.total_scan_time = int(lvl1_command.length() * 25 * (10 ** -9) * self.n_triggers)
disabled_pixels_limit_cnt = int(self.disabled_pixels_limit * 336 * 80)
preselected_pixels = invert_pixel_mask(self.register.get_pixel_register_value('Enable')).sum()
disabled_pixels = 0
for reg_val in range(scan_parameter_range[0], scan_parameter_range[1] - 1, -1):
if self.stop_run.is_set():
break
self.register.create_restore_point(name=str(reg_val))
logging.info('Scanning Vthin_AltFine %d', reg_val)
commands = []
commands.extend(self.register.get_commands("ConfMode"))
self.register.set_global_register_value("Vthin_AltFine", reg_val) # set number of consecutive triggers
commands.extend(self.register.get_commands("WrRegister", name=["Vthin_AltFine"]))
# setting FE into RunMode
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
step = 0
while True:
if self.stop_run.is_set():
break
self.histograming.reset()
step += 1
logging.info('Step %d / %d at Vthin_AltFine %d', step, steps, reg_val)
logging.info('Estimated scan time: %ds', self.total_scan_time)
with self.readout(Vthin_AltFine=reg_val, Step=step, reset_sram_fifo=True, fill_buffer=True, clear_buffer=True, callback=self.handle_data):
got_data = False
start = time()
self.register_utils.send_command(lvl1_command, repeat=self.n_triggers, wait_for_finish=False, set_length=True, clear_memory=False)
while not self.stop_run.wait(0.1):
if self.register_utils.is_ready:
if got_data:
self.progressbar.finish()
logging.info('Finished sending %d triggers', self.n_triggers)
break
if not got_data:
if self.fifo_readout.data_words_per_second() > 0:
got_data = True
logging.info('Taking data...')
self.progressbar = progressbar.ProgressBar(widgets=['', progressbar.Percentage(), ' ', progressbar.Bar(marker='*', left='|', right='|'), ' ', progressbar.Timer()], maxval=self.total_scan_time, poll=10, term_width=80).start()
else:
try:
self.progressbar.update(time() - start)
except ValueError:
pass
# Use fast C++ hit histograming to save time
raw_data = np.ascontiguousarray(data_array_from_data_iterable(self.fifo_readout.data), dtype=np.uint32)
self.interpreter.interpret_raw_data(raw_data)
self.interpreter.store_event() # force to create latest event
self.histograming.add_hits(self.interpreter.get_hits())
occ_hist = self.histograming.get_occupancy()[:, :, 0]
# noisy pixels are set to 1
occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
occ_mask[occ_hist > self.occupancy_limit * self.n_triggers * self.consecutive_lvl1] = 1
tdac_reg = self.register.get_pixel_register_value('TDAC')
decrease_pixel_mask = np.logical_and(occ_mask > 0, tdac_reg > 0)
disable_pixel_mask = np.logical_and(occ_mask > 0, tdac_reg == 0)
enable_reg = self.register.get_pixel_register_value('Enable')
enable_mask = np.logical_and(enable_reg, invert_pixel_mask(disable_pixel_mask))
if np.logical_and(occ_mask > 0, enable_reg == 0).sum():
logging.warning('Received data from disabled pixels')
# disabled_pixels += disable_pixel_mask.sum() # can lead to wrong values if the enable reg is corrupted
disabled_pixels = invert_pixel_mask(enable_mask).sum() - preselected_pixels
if disabled_pixels > disabled_pixels_limit_cnt:
logging.info('Limit of disabled pixels reached: %d (limit %d)... stopping scan' % (disabled_pixels, disabled_pixels_limit_cnt))
self.register.restore(name=str(reg_val))
break
else:
logging.info('Increasing threshold of %d pixel(s)', decrease_pixel_mask.sum())
logging.info('Disabling %d pixel(s), total number of disabled pixel(s): %d', disable_pixel_mask.sum(), disabled_pixels)
tdac_reg[decrease_pixel_mask] -= 1
self.register.set_pixel_register_value('TDAC', tdac_reg)
self.register.set_pixel_register_value('Enable', enable_mask)
commands = []
commands.extend(self.register.get_commands("ConfMode"))
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='TDAC'))
commands.extend(self.register.get_commands("WrFrontEnd", same_mask_for_all_dc=False, name='Enable'))
commands.extend(self.register.get_commands("RunMode"))
self.register_utils.send_commands(commands)
if occ_mask.sum() == 0 or step == steps or decrease_pixel_mask.sum() < disabled_pixels_limit_cnt:
self.register.clear_restore_points(name=str(reg_val))
self.last_tdac_distribution = self.register.get_pixel_register_value('TDAC')
self.last_occupancy_hist = occ_hist.copy()
self.last_occupancy_mask = occ_mask.copy()
self.last_reg_val = reg_val
self.last_step = step
break
else:
logging.info('Found %d noisy pixels... repeat tuning step for Vthin_AltFine %d', occ_mask.sum(), reg_val)
if disabled_pixels > disabled_pixels_limit_cnt:
self.last_good_threshold = self.register.get_global_register_value("Vthin_AltFine")
self.last_good_tdac = self.register.get_pixel_register_value('TDAC')
self.last_good_enable_mask = self.register.get_pixel_register_value('Enable')
break
def analyze(self):
self.register.set_global_register_value("Vthin_AltFine", self.last_good_threshold + self.increase_threshold)
self.register.set_pixel_register_value('TDAC', self.last_good_tdac)
self.register.set_pixel_register_value('Enable', self.last_good_enable_mask)
with AnalyzeRawData(raw_data_file=self.output_filename, create_pdf=True) as analyze_raw_data:
analyze_raw_data.create_source_scan_hist = True
analyze_raw_data.interpreter.set_warning_output(False)
analyze_raw_data.interpret_word_table()
analyze_raw_data.interpreter.print_summary()
analyze_raw_data.plot_histograms()
plot_occupancy(self.last_occupancy_hist.T, title='Noisy Pixels at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), filename=analyze_raw_data.output_pdf)
plot_fancy_occupancy(self.last_occupancy_hist.T, filename=analyze_raw_data.output_pdf)
plot_occupancy(self.last_occupancy_mask.T, title='Occupancy Mask at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), z_max=1, filename=analyze_raw_data.output_pdf)
plot_fancy_occupancy(self.last_occupancy_mask.T, filename=analyze_raw_data.output_pdf)
plot_three_way(self.last_tdac_distribution.T, title='TDAC at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), x_axis_title="TDAC", filename=analyze_raw_data.output_pdf, maximum=31, bins=32)
plot_occupancy(self.last_tdac_distribution.T, title='TDAC at Vthin_AltFine %d Step %d' % (self.last_reg_val, self.last_step), z_max=31, filename=analyze_raw_data.output_pdf)
plot_occupancy(self.register.get_pixel_register_value('Enable').T, title='Enable Mask', z_max=1, filename=analyze_raw_data.output_pdf)
plot_fancy_occupancy(self.register.get_pixel_register_value('Enable').T, filename=analyze_raw_data.output_pdf)
class DataWorker(QtCore.QObject):
connected = QtCore.pyqtSignal(bool)
data_ready = QtCore.pyqtSignal(tuple, list)
def __init__(self):
QtCore.QObject.__init__(self)
self.integrate_readouts = 0
self.n_readout = 0
self.setup_raw_data_analysis()
def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def connect(self, socket_addr):
self.socket_addr = socket_addr
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PULL)
self.socket.connect(self.socket_addr)
self.poller = zmq.Poller() # poll needed to be able to return QThread
self.poller.register(self.socket, zmq.POLLIN)
self.connected.emit(True)
def set_integrate_readouts(self, value):
self.integrate_readouts = value
def recv_data(self, flags=0, copy=True, track=False):
if self.poller.poll(10):
meta_data = self.socket.recv_json(flags=flags)
msg = self.socket.recv(flags=flags, copy=copy, track=track)
array = np.fromstring(msg, dtype=meta_data['dtype'])
return array.reshape(meta_data['shape']), meta_data['timestamp_start'], meta_data['timestamp_stop'], meta_data['readout_error'], re.sub(r'\bOrderedDict\b|[()\[\],\']', '', meta_data['scan_parameters'])
else:
return None, None, None, None, None
def analyze_raw_data(self, raw_data):
self.interpreter.interpret_raw_data(raw_data)
self.histograming.add_hits(self.interpreter.get_hits())
def process_data(self):
data = self.recv_data() # poll on ZMQ queue for data
if data[1]: # identify non empty data by existing time stamp
self.n_readout += 1
if self.integrate_readouts and self.n_readout % self.integrate_readouts == 0:
self.histograming.reset()
self.analyze_raw_data(data[0])
interpreted_data = self.histograming.get_occupancy(), self.histograming.get_tot_hist(), self.interpreter.get_tdc_counters(), self.interpreter.get_error_counters(), self.interpreter.get_service_records_counters(), self.interpreter.get_trigger_error_counters(), self.histograming.get_rel_bcid_hist()
self.data_ready.emit(interpreted_data, [data[1], data[2], data[3], data[4]])
QtCore.QTimer.singleShot(0.0001, self.process_data)
def __del__(self):
self.socket.close()
self.context.term()
class DataWorker(QtCore.QObject):
run_start = QtCore.pyqtSignal()
run_config_data = QtCore.pyqtSignal(dict)
global_config_data = QtCore.pyqtSignal(dict)
filename = QtCore.pyqtSignal(dict)
interpreted_data = QtCore.pyqtSignal(dict)
meta_data = QtCore.pyqtSignal(dict)
finished = QtCore.pyqtSignal()
def __init__(self):
QtCore.QObject.__init__(self)
self.integrate_readouts = 1
self.n_readout = 0
self._stop_readout = Event()
self.setup_raw_data_analysis()
self.reset_lock = Lock()
def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def connect(self, socket_addr):
self.socket_addr = socket_addr
self.context = zmq.Context()
self.socket_pull = self.context.socket(zmq.SUB) # subscriber
self.socket_pull.setsockopt(zmq.SUBSCRIBE, "") # do not filter any data
self.socket_pull.connect(self.socket_addr)
@pyqtSlot(float)
def on_set_integrate_readouts(self, value):
self.integrate_readouts = value
# @pyqtSlot()
def reset(self):
with self.reset_lock:
self.histograming.reset()
self.interpreter.reset()
self.n_readout = 0
def analyze_raw_data(self, raw_data):
self.interpreter.interpret_raw_data(raw_data)
self.histograming.add_hits(self.interpreter.get_hits())
@pyqtSlot()
def process_data(self): # infinite loop via QObject.moveToThread(), does not block event loop
while not self._stop_readout.wait(0.01): # use wait(), do not block here
with self.reset_lock:
try:
meta_data = self.socket_pull.recv_json(flags=zmq.NOBLOCK)
except zmq.Again:
pass
else:
name = meta_data.pop("name")
if name == "ReadoutData":
data = self.socket_pull.recv()
# reconstruct numpy array
buf = buffer(data)
dtype = meta_data.pop("dtype")
shape = meta_data.pop("shape")
data_array = np.frombuffer(buf, dtype=dtype).reshape(shape)
# count readouts and reset
self.n_readout += 1
if self.integrate_readouts != 0 and self.n_readout % self.integrate_readouts == 0:
self.histograming.reset()
# we do not want to reset interpreter to keep the error counters
# self.interpreter.reset()
# interpreted data
self.analyze_raw_data(data_array)
if (
self.integrate_readouts == 0
or self.n_readout % self.integrate_readouts == self.integrate_readouts - 1
):
interpreted_data = {
"occupancy": self.histograming.get_occupancy(),
"tot_hist": self.histograming.get_tot_hist(),
"tdc_counters": self.interpreter.get_tdc_counters(),
"error_counters": self.interpreter.get_error_counters(),
"service_records_counters": self.interpreter.get_service_records_counters(),
"trigger_error_counters": self.interpreter.get_trigger_error_counters(),
"rel_bcid_hist": self.histograming.get_rel_bcid_hist(),
}
self.interpreted_data.emit(interpreted_data)
# meta data
meta_data.update(
{"n_hits": self.interpreter.get_n_hits(), "n_events": self.interpreter.get_n_events()}
)
self.meta_data.emit(meta_data)
elif name == "RunConf":
self.run_config_data.emit(meta_data)
elif name == "GlobalRegisterConf":
trig_count = int(meta_data["conf"]["Trig_Count"])
self.interpreter.set_trig_count(trig_count)
self.global_config_data.emit(meta_data)
elif name == "Reset":
self.histograming.reset()
self.interpreter.reset()
self.run_start.emit()
elif name == "Filename":
self.filename.emit(meta_data)
self.finished.emit()
# @pyqtSlot()
def stop(self):
self._stop_readout.set()
def histogram_cluster_table(analyzed_data_file, output_file, chunk_size=10000000):
"""Reads in the cluster info table in chunks and histograms the seed pixels into one occupancy array.
The 3rd dimension of the occupancy array is the number of different scan parameters used
Parameters
----------
analyzed_data_file : hdf5 file containing the cluster table. If a scan parameter is given in the meta data the occupancy
histograming is done per scan parameter.
Returns
-------
occupancy_array: numpy.array with dimensions (col, row, #scan_parameter)
"""
with tb.openFile(analyzed_data_file, mode="r") as in_file_h5:
with tb.openFile(output_file, mode="w") as out_file_h5:
histograming = PyDataHistograming()
histograming.create_occupancy_hist(True)
scan_parameters = None
event_number_indices = None
scan_parameter_indices = None
try:
meta_data = in_file_h5.root.meta_data[:]
scan_parameters = analysis_utils.get_unique_scan_parameter_combinations(meta_data)
if scan_parameters is not None:
scan_parameter_indices = np.array(range(0, len(scan_parameters)), dtype="u4")
event_number_indices = np.ascontiguousarray(scan_parameters["event_number"]).astype(np.uint64)
histograming.add_meta_event_index(
event_number_indices, array_length=len(scan_parameters["event_number"])
)
histograming.add_scan_parameter(scan_parameter_indices)
logging.info("Add %d different scan parameter(s) for analysis", len(scan_parameters))
else:
logging.info("No scan parameter data provided")
histograming.set_no_scan_parameter()
except tb.exceptions.NoSuchNodeError:
logging.info("No meta data provided, use no scan parameter")
histograming.set_no_scan_parameter()
logging.info("Histogram cluster seeds...")
progress_bar = progressbar.ProgressBar(
widgets=[
"",
progressbar.Percentage(),
" ",
progressbar.Bar(marker="*", left="|", right="|"),
" ",
analysis_utils.ETA(),
],
maxval=in_file_h5.root.Cluster.shape[0],
term_width=80,
)
progress_bar.start()
total_cluster = 0 # to check analysis
for cluster, index in analysis_utils.data_aligned_at_events(in_file_h5.root.Cluster, chunk_size=chunk_size):
total_cluster += len(cluster)
histograming.add_cluster_seed_hits(cluster, len(cluster))
progress_bar.update(index)
progress_bar.finish()
filter_table = tb.Filters(complib="blosc", complevel=5, fletcher32=False) # compression of the written data
occupancy_array = histograming.get_occupancy().T
occupancy_array_table = out_file_h5.createCArray(
out_file_h5.root,
name="HistOcc",
title="Occupancy Histogram",
atom=tb.Atom.from_dtype(occupancy_array.dtype),
shape=occupancy_array.shape,
filters=filter_table,
)
occupancy_array_table[:] = occupancy_array
if total_cluster != np.sum(occupancy_array):
logging.warning("Analysis shows inconsistent number of cluster used. Check needed!")
in_file_h5.root.meta_data.copy(out_file_h5.root) # copy meta_data note to new file
def histogram_cluster_table(analyzed_data_file,
output_file,
chunk_size=10000000):
'''Reads in the cluster info table in chunks and histograms the seed pixels into one occupancy array.
The 3rd dimension of the occupancy array is the number of different scan parameters used
Parameters
----------
analyzed_data_file : hdf5 file containing the cluster table. If a scan parameter is given in the meta data the occupancy
histograming is done per scan parameter.
Returns
-------
occupancy_array: numpy.array with dimensions (col, row, #scan_parameter)
'''
with tb.openFile(analyzed_data_file, mode="r") as in_file_h5:
with tb.openFile(output_file, mode="w") as out_file_h5:
histograming = PyDataHistograming()
histograming.create_occupancy_hist(True)
scan_parameters = None
event_number_indices = None
scan_parameter_indices = None
try:
meta_data = in_file_h5.root.meta_data[:]
scan_parameters = analysis_utils.get_unique_scan_parameter_combinations(
meta_data)
if scan_parameters is not None:
scan_parameter_indices = np.array(range(
0, len(scan_parameters)),
dtype='u4')
event_number_indices = np.ascontiguousarray(
scan_parameters['event_number']).astype(np.uint64)
histograming.add_meta_event_index(
event_number_indices,
array_length=len(scan_parameters['event_number']))
histograming.add_scan_parameter(scan_parameter_indices)
logging.info(
"Add %d different scan parameter(s) for analysis",
len(scan_parameters))
else:
logging.info("No scan parameter data provided")
histograming.set_no_scan_parameter()
except tb.exceptions.NoSuchNodeError:
logging.info("No meta data provided, use no scan parameter")
histograming.set_no_scan_parameter()
logging.info('Histogram cluster seeds...')
progress_bar = progressbar.ProgressBar(
widgets=[
'',
progressbar.Percentage(), ' ',
progressbar.Bar(marker='*', left='|', right='|'), ' ',
analysis_utils.ETA()
],
maxval=in_file_h5.root.Cluster.shape[0],
term_width=80)
progress_bar.start()
total_cluster = 0 # to check analysis
for cluster, index in analysis_utils.data_aligned_at_events(
in_file_h5.root.Cluster, chunk_size=chunk_size):
total_cluster += len(cluster)
histograming.add_cluster_seed_hits(cluster, len(cluster))
progress_bar.update(index)
progress_bar.finish()
filter_table = tb.Filters(
complib='blosc', complevel=5,
fletcher32=False) # compression of the written data
occupancy_array = histograming.get_occupancy().T
occupancy_array_table = out_file_h5.createCArray(
out_file_h5.root,
name='HistOcc',
title='Occupancy Histogram',
atom=tb.Atom.from_dtype(occupancy_array.dtype),
shape=occupancy_array.shape,
filters=filter_table)
occupancy_array_table[:] = occupancy_array
if total_cluster != np.sum(occupancy_array):
logging.warning(
'Analysis shows inconsistent number of cluster used. Check needed!'
)
in_file_h5.root.meta_data.copy(
out_file_h5.root) # copy meta_data note to new file
class DataWorker(QtCore.QObject):
run_start = QtCore.pyqtSignal()
run_config_data = QtCore.pyqtSignal(dict)
global_config_data = QtCore.pyqtSignal(dict)
filename = QtCore.pyqtSignal(dict)
interpreted_data = QtCore.pyqtSignal(dict)
meta_data = QtCore.pyqtSignal(dict)
finished = QtCore.pyqtSignal()
def __init__(self):
QtCore.QObject.__init__(self)
self.integrate_readouts = 1
self.n_readout = 0
self._stop_readout = Event()
self.setup_raw_data_analysis()
self.reset_lock = Lock()
def setup_raw_data_analysis(self):
self.interpreter = PyDataInterpreter()
self.histograming = PyDataHistograming()
self.interpreter.set_warning_output(False)
self.histograming.set_no_scan_parameter()
self.histograming.create_occupancy_hist(True)
self.histograming.create_rel_bcid_hist(True)
self.histograming.create_tot_hist(True)
self.histograming.create_tdc_hist(True)
def connect(self, socket_addr):
self.socket_addr = socket_addr
self.context = zmq.Context()
self.socket_pull = self.context.socket(zmq.SUB) # subscriber
self.socket_pull.setsockopt(zmq.SUBSCRIBE,
'') # do not filter any data
self.socket_pull.connect(self.socket_addr)
@pyqtSlot(float)
def on_set_integrate_readouts(self, value):
self.integrate_readouts = value
# @pyqtSlot()
def reset(self):
with self.reset_lock:
self.histograming.reset()
self.interpreter.reset()
self.n_readout = 0
def analyze_raw_data(self, raw_data):
self.interpreter.interpret_raw_data(raw_data)
self.histograming.add_hits(self.interpreter.get_hits())
@pyqtSlot()
def process_data(
self
): # infinite loop via QObject.moveToThread(), does not block event loop
while (not self._stop_readout.wait(0.01)
): # use wait(), do not block here
with self.reset_lock:
try:
meta_data = self.socket_pull.recv_json(flags=zmq.NOBLOCK)
except zmq.Again:
pass
else:
name = meta_data.pop('name')
if name == 'ReadoutData':
data = self.socket_pull.recv()
# reconstruct numpy array
buf = buffer(data)
dtype = meta_data.pop('dtype')
shape = meta_data.pop('shape')
data_array = np.frombuffer(buf,
dtype=dtype).reshape(shape)
# count readouts and reset
self.n_readout += 1
if self.integrate_readouts != 0 and self.n_readout % self.integrate_readouts == 0:
self.histograming.reset()
# we do not want to reset interpreter to keep the error counters
# self.interpreter.reset()
# interpreted data
self.analyze_raw_data(data_array)
if self.integrate_readouts == 0 or self.n_readout % self.integrate_readouts == self.integrate_readouts - 1:
interpreted_data = {
'occupancy':
self.histograming.get_occupancy(),
'tot_hist':
self.histograming.get_tot_hist(),
'tdc_counters':
self.interpreter.get_tdc_counters(),
'error_counters':
self.interpreter.get_error_counters(),
'service_records_counters':
self.interpreter.get_service_records_counters(
),
'trigger_error_counters':
self.interpreter.get_trigger_error_counters(),
'rel_bcid_hist':
self.histograming.get_rel_bcid_hist()
}
self.interpreted_data.emit(interpreted_data)
# meta data
meta_data.update({
'n_hits':
self.interpreter.get_n_hits(),
'n_events':
self.interpreter.get_n_events()
})
self.meta_data.emit(meta_data)
elif name == 'RunConf':
self.run_config_data.emit(meta_data)
elif name == 'GlobalRegisterConf':
trig_count = int(meta_data['conf']['Trig_Count'])
self.interpreter.set_trig_count(trig_count)
self.global_config_data.emit(meta_data)
elif name == 'Reset':
self.histograming.reset()
self.interpreter.reset()
self.run_start.emit()
elif name == 'Filename':
self.filename.emit(meta_data)
self.finished.emit()
# @pyqtSlot()
def stop(self):
self._stop_readout.set()
