def _rolling_file(self):
f_size = os.path.getsize(self._file_path)
if f_size > self._file_size:
# self.set_file_path(self._create_new_file_name(self._file_path))
EventLogger.info(
"Max Filesize(" + "%.3f" % (self._file_size / 1024.0 / 1024.0) + " MB) reached! Rolling Files...")
self._roll_files()
def _btn_add_device_clicked(self):
"""
Add the selected device from the list/DataLogger-Config.
"""
items = self.list_widget.selectedItems()
cur_dev = GuiConfigHandler.get_simple_blueprint(self.Ui_Logger)
for item in items:
name = item.text()
if name == self._no_connected_device_string or name == self._list_separator_string:
# ignore those
continue
dev_name, uid = Utilities.parse_device_name(name)
dev = GuiConfigHandler.get_device_blueprint(dev_name)
if dev is None:
EventLogger.debug("DeviceDialog._btn_add_device_clicked: Blueprint(" + str(dev_name) + ") was None!")
continue
if uid is not None:
if self.__is_device_in_list(dev_name, uid, cur_dev):
continue
# else
dev[Identifier.DD_UID] = uid
else:
dev[Identifier.DD_UID] = Identifier.DD_UID_DEFAULT
self._logger_window.add_item_to_tree(dev)
def update_setup_tab(self, config):
EventLogger.debug('Updating setup tab from config')
name = config['hosts']['default']['name']
port = config['hosts']['default']['port']
secret = config['hosts']['default']['secret']
i = self.combo_host.findText(name)
if i >= 0:
self.combo_host.setCurrentIndex(i)
else:
self.combo_host.insertItem(0, name, (port, secret != None, secret))
self.combo_host.setCurrentIndex(0)
self.spin_port.setValue(port)
self.check_authentication.setChecked(secret != None)
self.edit_secret.setText(secret if secret != None else '')
self.combo_data_time_format.setCurrentIndex(max(self.combo_data_time_format.findData(config['data']['time_format']), 0))
self.edit_data_time_format_strftime.setText(config['data']['time_format_strftime'])
self.check_data_to_csv_file.setChecked(config['data']['csv']['enabled'])
self.edit_csv_file_name.setText(config['data']['csv']['file_name'])
self.combo_debug_time_format.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['time_format']), 0))
self.check_debug_to_log_file.setChecked(config['debug']['log']['enabled'])
self.edit_log_file_name.setText(config['debug']['log']['file_name'])
self.combo_log_level.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['log']['level']), 0))
def __init__(self, path_to_config=None, configuration=None):
"""
pathToConfig -- path to the json configuration file
OR
configuration -- the configuration itself
"""
self._configuration = Configuration()
self._readConfigErr = 0 # Errors which occure during readin
if path_to_config is None and configuration is None:
EventLogger.critical(
"ConfigurationReader needs a path to the configuration file or an actual configuration")
return
if path_to_config is not None:
self.fileName = path_to_config
self._read_json_config_file()
if configuration is not None:
if isinstance(configuration, Configuration):
self._configuration = configuration
else:
self.map_dict_to_config(configuration)
validator = ConfigurationValidator(self._configuration)
validator._error_count += self._readConfigErr
validator.validate()
def validate_xively_section(self):
"""
This function validates the xively section out of the configuration
"""
# TODO: implement xively section validation
# xively_section = self.json_config._xively
EventLogger.warning("Xively validation is not yet supported")
def validate_xively_section(self):
"""
This function validates the xively section out of the configuration
"""
# TODO: implement xively section validation
# xively_section = self.json_config._xively
EventLogger.warning("Xively validation is not yet supported")
def _job(self):
try:
# check for datalogger object
if AbstractJob._job(self):
return
EventLogger.debug(self._job_name + " Started")
while True:
if not self._datalogger.data_queue[self.name].empty():
csv_data = self._get_data_from_queue()
self.signalNewData.emit(csv_data)
if not self._exit_flag and self._datalogger.data_queue[
self.name].empty():
time.sleep(self._datalogger.job_sleep)
if self._exit_flag and self._datalogger.data_queue[
self.name].empty():
self._remove_from_data_queue()
break
except Exception as e:
EventLogger.critical(self._job_name + " -.- " + str(e))
self.stop()
def btn_remove_device_clicked(self):
"""
Removes selected Device
"""
selected_item = self.tree_devices.selectedItems()
for index in range(0, len(selected_item)):
try:
if selected_item[index] is None:
continue
device_name = selected_item[index].text(0)
device_id = selected_item[index].text(1)
if selected_item[index].text(0) not in Identifier.DEVICE_DEFINITIONS:
# have to find the parent
current_item = selected_item[0]
while True:
if current_item.parent() is None:
if current_item.text(0) not in Identifier.DEVICE_DEFINITIONS:
EventLogger.error("Cant remove device: " + selected_item[index].text(0))
device_name = ""
device_id = ""
break
else:
device_name = current_item.text(0)
device_id = current_item.text(1)
break
else:
current_item = current_item.parent()
self.remove_item_from_tree(device_name, device_id)
except Exception as e:
if not str(e).startswith("wrapped C/C++ object"):
EventLogger.error("Cant remove device: " + str(e)) # was already removed
def validate_devices_section(self):
"""
This function validates the devices out of the configuration file
:return:
"""
device_definitions = Idf.DEVICE_DEFINITIONS
for device in self.json_config._devices:
# name
blueprint = device_definitions[device[Idf.DD_NAME]]
if blueprint is None:
EventLogger.critical(
self._generate_device_error_message(uid=device[Idf.DD_UID],
tier_array=["general"], msg="no such device available"))
continue # next device
# uid
if not Utilities.is_valid_string(device[Idf.DD_UID], 3) or device[Idf.DD_UID] == Idf.DD_UID_DEFAULT:
EventLogger.critical(
self._generate_device_error_message(uid=device[Idf.DD_UID],
tier_array=["general"], msg="the UID from '"+device[Idf.DD_NAME]+"' is invalid"))
device_values = device[Idf.DD_VALUES]
blueprint_values = blueprint[Idf.DD_VALUES]
# values
for device_value in device_values:
logged_values = 0
if device_value not in blueprint_values:
EventLogger.critical(
self._generate_device_error_message(uid=device[Idf.DD_UID],
tier_array=["values"],
msg="invalid value " + str(device_value)))
else:
# interval
interval = device_values[device_value][Idf.DD_VALUES_INTERVAL]
if not self._is_valid_interval(interval):
EventLogger.critical(
self._generate_device_error_message(uid=device[Idf.DD_UID],
tier_array=["values"],
msg="invalid interval " + str(interval)))
# subvalue
try:
subvalues = device_values[device_value][Idf.DD_SUBVALUES]
for value in subvalues:
if not type(subvalues[value]) == bool: # type check for validation
EventLogger.critical(
self._generate_device_error_message(
uid=device[Idf.DD_UID],
tier_array=["values"],
msg="invalid type " + str(value)))
else:
if subvalues[value]: # value check for "lines per second" calculation
logged_values += 1
except KeyError:
if interval > 0: # just one value to log
logged_values += 1
if interval > 0:
self._log_space_counter.add_lines_per_second(interval / 1000 * logged_values)
def __init__(self, parent):
QDialog.__init__(self, parent)
self.setWindowFlags(Qt.Window | Qt.WindowCloseButtonHint)
self._gui_logger = GUILogger("GUILogger", EventLogger.EVENT_LOG_LEVEL)
self._gui_job = None
EventLogger.add_logger(self._gui_logger)
# FIXME better way to find interval and uids in tree_widget?!
self.__tree_interval_tooltip = "Interval in milliseconds"
self.__tree_uid_tooltip = "UID must be at least 3 Character long"
self.data_logger_thread = None
self.tab_console_warning = False
self.logger_device_dialog = None
# Code Inspector
self.host_infos = None
self.last_host = None
self.host_index_changing = None
# if self._table_widget is not None:#FIXME rework this like the console_tab <-- what does that mean?!
# self.jobs.append()
self.setupUi(self)
self.widget_initialization()
def validate(self):
"""
This function performs the validation of the various sections of the json
configuration file
"""
EventLogger.info("Started configuration file validation")
self.validate_general_section()
self.validate_xively_section()
self.validate_devices_section()
EventLogger.info("Validation ends with [" + str(self._error_count) +
"] errors")
logging_time = self._log_space_counter.calculate_time()
if self._log_space_counter.file_size != 0:
EventLogger.info(
"Logging time until old data will be overwritten.")
EventLogger.info("Days: " + str(logging_time[0]) + " Hours: " +
str(logging_time[1]) + " Minutes: " +
str(logging_time[2]) + " Seconds: " +
str(logging_time[3]))
EventLogger.info(
"Will write about " +
str(int(self._log_space_counter.lines_per_second + 0.5)) +
" lines per second into the log-file.")
if self._error_count != 0:
raise DataLoggerException(DataLoggerException.DL_FAILED_VALIDATION,
"Validation process found some errors")
def signal_handler(interrupted_ref, signum, frame):
"""
This function handles the ctrl + c exit condition
if it's raised through the console
"""
EventLogger.info('Received SIGINT/SIGTERM')
interrupted_ref[0] = True
def update_setup_tab(self, config):
EventLogger.debug('Updating setup tab from config')
self.combo_host.setEditText(config['hosts']['default']['name'])
self.spin_port.setValue(config['hosts']['default']['port'])
self.combo_data_time_format.setCurrentIndex(
max(
self.combo_data_time_format.findData(
config['data']['time_format']), 0))
self.check_data_to_csv_file.setChecked(
config['data']['csv']['enabled'])
self.edit_csv_file_name.setText(
config['data']['csv']['file_name'].decode('utf-8'))
self.combo_debug_time_format.setCurrentIndex(
max(
self.combo_debug_time_format.findData(
config['debug']['time_format']), 0))
self.check_debug_to_log_file.setChecked(
config['debug']['log']['enabled'])
self.edit_log_file_name.setText(
config['debug']['log']['file_name'].decode('utf-8'))
self.combo_log_level.setCurrentIndex(
max(
self.combo_debug_time_format.findData(
config['debug']['log']['level']), 0))
def __init__(self, parent):
QDialog.__init__(self, parent)
self.setWindowFlags(Qt.Window | Qt.WindowCloseButtonHint)
self._gui_logger = GUILogger("GUILogger", EventLogger.EVENT_LOG_LEVEL)
self._gui_job = None
EventLogger.add_logger(self._gui_logger)
# FIXME better way to find interval and uids in tree_widget?!
self.__tree_interval_tooltip = "Interval in milliseconds"
self.__tree_uid_tooltip = "UID must be at least 3 Character long"
self.data_logger_thread = None
self.tab_console_warning = False
self.logger_device_dialog = None
# Code Inspector
self.host_infos = None
self.last_host = None
self.host_index_changing = None
# if self._table_widget is not None:#FIXME rework this like the console_tab <-- what does that mean?!
# self.jobs.append()
self.setupUi(self)
self.widget_initialization()
def __init__(self, path_to_config=None, configuration=None):
"""
pathToConfig -- path to the json configuration file
OR
configuration -- the configuration itself
"""
self._configuration = Configuration()
self._readConfigErr = 0 # Errors which occure during readin
if path_to_config is None and configuration is None:
EventLogger.critical(
"ConfigurationReader needs a path to the configuration file or an actual configuration"
)
return
if path_to_config is not None:
self.fileName = path_to_config
self._read_json_config_file()
if configuration is not None:
if isinstance(configuration, Configuration):
self._configuration = configuration
else:
self.map_dict_to_config(configuration)
validator = ConfigurationValidator(self._configuration)
validator._error_count += self._readConfigErr
validator.validate()
def _remove_from_data_queue(self):
try:
self._datalogger.data_queue.pop(self.name)
except KeyError as key_err:
EventLogger.warning("Job:" + self.name +
" was not in the DataQueue! -> " +
str(key_err))
def update_devices_tab(self, config):
EventLogger.debug('Updating devices tab from config')
self.model_devices.removeRows(0, self.model_data.rowCount())
for device in config['devices']:
self.add_device_to_tree(device)
def signal_handler(interrupted_ref, signum, frame):
"""
This function handles the ctrl + c exit condition
if it's raised through the console
"""
EventLogger.info('Received SIGINT/SIGTERM')
interrupted_ref[0] = True
def update_setup_tab(self, config):
EventLogger.debug('Updating setup tab from config')
name = config['hosts']['default']['name']
port = config['hosts']['default']['port']
secret = config['hosts']['default']['secret']
i = self.combo_host.findText(name)
if i >= 0:
self.combo_host.setCurrentIndex(i)
else:
self.combo_host.insertItem(0, name, (port, secret != None, secret))
self.combo_host.setCurrentIndex(0)
self.spin_port.setValue(port)
self.check_authentication.setChecked(secret != None)
self.edit_secret.setText(secret if secret != None else '')
self.combo_data_time_format.setCurrentIndex(max(self.combo_data_time_format.findData(config['data']['time_format']), 0))
self.edit_data_time_format_strftime.setText(config['data']['time_format_strftime'])
self.check_data_to_csv_file.setChecked(config['data']['csv']['enabled'])
self.edit_csv_file_name.setText(config['data']['csv']['file_name'])
self.combo_debug_time_format.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['time_format']), 0))
self.check_debug_to_log_file.setChecked(config['debug']['log']['enabled'])
self.edit_log_file_name.setText(config['debug']['log']['file_name'])
self.combo_log_level.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['log']['level']), 0))
def _rolling_file(self):
f_size = os.path.getsize(self._file_path)
if f_size > self._file_size:
EventLogger.info("Max Filesize(" + "%.3f" %
(self._file_size / 1024.0 / 1024.0) +
" MB) reached! Rolling Files...")
self._roll_files()
def update_devices_tab(self, config):
EventLogger.debug('Updating devices tab from config')
self.model_devices.removeRows(0, self.model_data.rowCount())
for device in config['devices']:
self.add_device_to_tree(device)
def main(config_filename, gui_config, gui_job, override_csv_file_name,
override_log_file_name, interrupted_ref):
"""
This function initialize the data logger and starts the logging process
"""
config = None
gui_start = False
if config_filename != None: # started via console
config = load_and_validate_config(config_filename)
if config == None:
return None
else: # started via GUI
config = gui_config
gui_start = True
if override_csv_file_name != None:
config['data']['csv']['file_name'] = override_csv_file_name
if override_log_file_name != None:
config['debug']['log']['file_name'] = override_log_file_name
try:
if config['debug']['log']['enabled']:
EventLogger.add_logger(
FileLogger(
'FileLogger',
log_level_name_to_id(config['debug']['log']['level']),
config['debug']['log']['file_name']))
data_logger = DataLogger(config, gui_job)
if data_logger.ipcon is not None:
data_logger.run()
if not gui_start:
while not interrupted_ref[0]:
try:
time.sleep(0.25)
except:
pass
data_logger.stop()
sys.exit(0)
else:
raise DataLoggerException(
DataLoggerException.DL_CRITICAL_ERROR,
"DataLogger did not start logging process! Please check for errors."
)
except Exception as exc:
EventLogger.critical(str(exc))
if gui_start:
return None
else:
sys.exit(DataLoggerException.DL_CRITICAL_ERROR)
return data_logger
def update_setup_tab(self, general_section):
"""
Update the information of the setup tab with the given general_section.
"""
from brickv.data_logger.configuration_validator import ConfigurationReader
try:
# host combo_host setEditText(String)
self.combo_host.setEditText(
general_section[ConfigurationReader.GENERAL_HOST])
# port spinbox_port setValue(int)
self.spinbox_port.setValue(
general_section[ConfigurationReader.GENERAL_PORT])
# file_count spin_file_count setValue(int)
self.spin_file_count.setValue(
general_section[ConfigurationReader.GENERAL_LOG_COUNT])
# file_size spin_file_size setValue(int/1024/1024) (Byte -> MB)
self.spin_file_size.setValue(
(general_section[ConfigurationReader.GENERAL_LOG_FILE_SIZE] /
1024.0 / 1024.0))
# path_to_file line_path_to_file setText(string)
self.line_path_to_file.setText(
general_section[ConfigurationReader.GENERAL_PATH_TO_FILE])
# logfile path
self.line_path_to_eventfile.setText(
general_section[ConfigurationReader.GENERAL_EVENTLOG_PATH])
# loglevel
ll = general_section[ConfigurationReader.GENERAL_EVENTLOG_LEVEL]
od = collections.OrderedDict(
sorted(GUILogger._convert_level.items()))
counter = 0 # TODO better way to set the combo box index?
for k in od.keys():
if ll == k:
break
counter += 1
self.combo_loglevel.setCurrentIndex(counter)
# log_to_console
def __checkbox_bool_setter(bool_value):
if bool_value:
return 2
else:
return 0
self.checkbox_to_file.setChecked(
__checkbox_bool_setter(general_section[
ConfigurationReader.GENERAL_EVENTLOG_TO_FILE]))
# log_to_file
self.checkbox_to_console.setCheckState(
__checkbox_bool_setter(general_section[
ConfigurationReader.GENERAL_EVENTLOG_TO_CONSOLE]))
except Exception as e:
EventLogger.critical(
"Could not read the General Section of the Config-File! -> " +
str(e))
return
def _job(self):
# check for datalogger object
if self._datalogger is None:
EventLogger.warning(
self.name +
" started but did not get a DataLogger Object! No work could be done."
)
return True
return False
def _write_header(self):
"""Writes a csv header into the file"""
if not self._file_is_empty():
EventLogger.debug("File is not empty")
return
EventLogger.debug("CSVWriter._write_header() - done")
self._csv_file.writerow(["TIME"] + ["NAME"] + ["UID"] + ["VAR"] + ["RAW"] + ["UNIT"])
self._raw_file.flush()
def __init__(self,
datalogger=None,
group=None,
name="XivelyJob",
args=(),
kwargs=None,
verbose=None):
super(XivelyJob, self).__init__()
EventLogger.warning(self._job_name + " Is not supported!")
raise Exception("XivelyJob not yet implemented!")
def main(config_filename, gui_config, gui_job, override_csv_file_name,
override_log_file_name, interrupted_ref):
"""
This function initialize the data logger and starts the logging process
"""
config = None
gui_start = False
if config_filename != None: # started via console
config = load_and_validate_config(config_filename)
if config == None:
return None
else: # started via GUI
config = gui_config
gui_start = True
if override_csv_file_name != None:
config['data']['csv']['file_name'] = override_csv_file_name
if override_log_file_name != None:
config['debug']['log']['file_name'] = override_log_file_name
if config['debug']['log']['enabled']:
EventLogger.add_logger(FileLogger('FileLogger', log_level_name_to_id(config['debug']['log']['level']),
config['debug']['log']['file_name']))
try:
data_logger = DataLogger(config, gui_job)
if data_logger.ipcon is not None:
data_logger.run()
if not gui_start:
while not interrupted_ref[0]:
try:
time.sleep(0.25)
except:
pass
data_logger.stop()
sys.exit(0)
else:
raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR,
"DataLogger did not start logging process! Please check for errors.")
except Exception as exc:
EventLogger.critical(str(exc))
if gui_start:
return None
else:
sys.exit(DataLoggerException.DL_CRITICAL_ERROR)
return data_logger
def process_data_csv_section(self):
"""
Information out of the general section will be consumed here
"""
csv = self._config['data']['csv']
self.csv_enabled = csv['enabled']
self.csv_file_name = csv['file_name']
if self.csv_enabled:
EventLogger.info("Logging data to CSV file: " + str(self.csv_file_name))
def process_data_csv_section(self):
"""
Information out of the general section will be consumed here
"""
csv = self._config['data']['csv']
self.csv_enabled = csv['enabled']
self.csv_file_name = csv['file_name']
EventLogger.debug("Logging output to CSV file: " + str(self.csv_enabled))
EventLogger.debug("Output file path: " + str(self.csv_file_name))
def prevent_key_error(dict_src, key):
"""
This function returns an empty array if there is no such
section in the configuration file
key -- section key
"""
result = []
try:
result = dict_src[key]
except KeyError:
EventLogger.warning("json configuration file has no [" + key + "] section")
return result
def process_data_csv_section(self):
"""
Information out of the general section will be consumed here
"""
csv = self._config['data']['csv']
self.csv_enabled = csv['enabled']
self.csv_file_name = csv['file_name']
if self.csv_enabled:
EventLogger.info("Logging data to CSV file: " +
str(self.csv_file_name))
def btn_set_eventfile_clicked(self):
"""
Opens a FileSelectionDialog and sets the selected path for the event output file.
"""
fn = self.__choose_file_dialog('Choose Eventfile destination', "Log-Files (*.log)")
if fn == "":
# cancel
EventLogger.debug("Cancelled select Eventfile-Path.")
return
self.line_path_to_eventfile.setText(fn)
def prevent_key_error(dict_src, key):
"""
This function returns an empty array if there is no such
section in the configuration file
key -- section key
"""
result = []
try:
result = dict_src[key]
except KeyError:
EventLogger.warning("json configuration file has no [" + key +
"] section")
return result
def btn_set_eventfile_clicked(self):
"""
Opens a FileSelectionDialog and sets the selected path for the event output file.
"""
fn = self.__choose_file_dialog('Choose Eventfile destination',
"Log-Files (*.log)")
if fn == "":
# cancel
EventLogger.debug("Cancelled select Eventfile-Path.")
return
self.line_path_to_eventfile.setText(fn)
def btn_set_logfile_clicked(self):
"""
Opens a FileSelectionDialog and sets the selected path for the data output file.
"""
fn = self.__choose_file_dialog('Choose Config Destination', "CSV-Files (*.csv);;Text-Files (*.txt)")
if fn == "":
# cancel
self.line_path_to_file.setText("")
EventLogger.debug("Cancelled select Config-File-Path.")
return
self.line_path_to_file.setText(fn)
def process_general_section(self):
"""
Information out of the general section will be consumed here
"""
data = self._configuration._general
self.log_to_file = data[ConfigurationReader.GENERAL_LOG_TO_FILE]
self.default_file_path = data[ConfigurationReader.GENERAL_PATH_TO_FILE]
self.max_file_size = data[ConfigurationReader.GENERAL_LOG_FILE_SIZE]
self.max_file_count = data[ConfigurationReader.GENERAL_LOG_COUNT]
EventLogger.debug("Logging output to file: " + str(self.log_to_file))
EventLogger.debug("Output file path: " + str(self.default_file_path))
def update_setup_tab(self, config):
EventLogger.debug('Updating setup tab from config')
self.combo_host.setEditText(config['hosts']['default']['name'])
self.spin_port.setValue(config['hosts']['default']['port'])
self.combo_data_time_format.setCurrentIndex(max(self.combo_data_time_format.findData(config['data']['time_format']), 0))
self.check_data_to_csv_file.setChecked(config['data']['csv']['enabled'])
self.edit_csv_file_name.setText(config['data']['csv']['file_name'].decode('utf-8'))
self.combo_debug_time_format.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['time_format']), 0))
self.check_debug_to_log_file.setChecked(config['debug']['log']['enabled'])
self.edit_log_file_name.setText(config['debug']['log']['file_name'].decode('utf-8'))
self.combo_log_level.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['log']['level']), 0))
def btn_load_config_clicked(self):
"""
Opens a FileSelectionDialog and loads the selected config.
"""
fn = QtGui.QFileDialog.getOpenFileName(self, "Open Config-File...", os.getcwd(), "JSON-Files (*.json)")
if fn == "":
# cancel
EventLogger.debug("Cancelled save Config.")
return
config_json = None
try:
with codecs.open(fn, 'r', 'UTF-8') as content_file:
try:
config_json = json.load(content_file)
except ValueError as e:
EventLogger.warning("Load Config - Cant parse the configuration file: " + str(e))
except Exception as e1:
EventLogger.warning("Load Config - Exception: " + str(e1))
return
EventLogger.info("Loaded Config-File from: " + str(fn))
# devices
config_blueprint = GuiConfigHandler.load_devices(config_json)
if config_blueprint is None:
return
self.create_tree_items(config_blueprint)
# general_section
from brickv.data_logger.configuration_validator import ConfigurationReader
self.update_setup_tab(config_json[ConfigurationReader.GENERAL_SECTION])
def stop(self):
"""
This function ends the logging process. self.stopped will be set to True if
the data logger stops
"""
EventLogger.info("Closing Timers and Threads...")
"""CLEANUP_AFTER_STOP """
# check if all timers stopped
for t in self.timers:
t.stop()
for t in self.timers:
t.join()
EventLogger.debug("Get-Timers[" + str(len(self.timers)) + "] stopped.")
# set THREAD_EXIT_FLAG for all work threads
for job in self.jobs:
job.stop()
# wait for all threads to stop
for job in self.jobs:
job.join()
EventLogger.debug("Jobs[" + str(len(self.jobs)) + "] stopped.")
try:
self.ipcon.disconnect()
except:
pass
EventLogger.info("Connection closed successfully.")
self.stopped = True
def stop(self):
"""
This function ends the logging process. self.stopped will be set to True if
the data logger stops
"""
EventLogger.info("Closing Timers and Threads...")
"""CLEANUP_AFTER_STOP """
# check if all timers stopped
for t in self.timers:
t.stop()
for t in self.timers:
t.join()
EventLogger.debug("Get-Timers[" + str(len(self.timers)) + "] stopped.")
# set THREAD_EXIT_FLAG for all work threads
for job in self.jobs:
job.stop()
# wait for all threads to stop
for job in self.jobs:
job.join()
EventLogger.debug("Jobs[" + str(len(self.jobs)) + "] stopped.")
try:
self.ipcon.disconnect()
except:
pass
EventLogger.info("Connection closed successfully.")
self.stopped = True
def btn_set_logfile_clicked(self):
"""
Opens a FileSelectionDialog and sets the selected path for the data output file.
"""
fn = self.__choose_file_dialog(
'Choose Config Destination',
"CSV-Files (*.csv);;Text-Files (*.txt)")
if fn == "":
# cancel
self.line_path_to_file.setText("")
EventLogger.debug("Cancelled select Config-File-Path.")
return
self.line_path_to_file.setText(fn)
def update_setup_tab(self, general_section):
"""
Update the information of the setup tab with the given general_section.
"""
from brickv.data_logger.configuration_validator import ConfigurationReader
try:
# host combo_host setEditText(String)
self.combo_host.setEditText(general_section[ConfigurationReader.GENERAL_HOST])
# port spinbox_port setValue(int)
self.spinbox_port.setValue(general_section[ConfigurationReader.GENERAL_PORT])
# file_count spin_file_count setValue(int)
self.spin_file_count.setValue(general_section[ConfigurationReader.GENERAL_LOG_COUNT])
# file_size spin_file_size setValue(int/1024/1024) (Byte -> MB)
self.spin_file_size.setValue((general_section[ConfigurationReader.GENERAL_LOG_FILE_SIZE] / 1024.0 / 1024.0))
# path_to_file line_path_to_file setText(string)
self.line_path_to_file.setText(general_section[ConfigurationReader.GENERAL_PATH_TO_FILE])
# logfile path
self.line_path_to_eventfile.setText(general_section[ConfigurationReader.GENERAL_EVENTLOG_PATH])
# loglevel
ll = general_section[ConfigurationReader.GENERAL_EVENTLOG_LEVEL]
od = collections.OrderedDict(sorted(GUILogger._convert_level.items()))
counter = 0 # TODO better way to set the combo box index?
for k in od.keys():
if ll == k:
break
counter += 1
self.combo_loglevel.setCurrentIndex(counter)
# log_to_console
def __checkbox_bool_setter(bool_value):
if bool_value:
return 2
else:
return 0
self.checkbox_to_file.setChecked(
__checkbox_bool_setter(general_section[ConfigurationReader.GENERAL_EVENTLOG_TO_FILE]))
# log_to_file
self.checkbox_to_console.setCheckState(
__checkbox_bool_setter(general_section[ConfigurationReader.GENERAL_EVENTLOG_TO_CONSOLE]))
except Exception as e:
EventLogger.critical("Could not read the General Section of the Config-File! -> " + str(e))
return
def _read_json_config_file(self):
with codecs.open(self.fileName, 'r', 'UTF-8') as content_file:
try:
json_structure = json.load(content_file)
except ValueError as e:
EventLogger.critical("Cant parse the configuration file: " + str(e))
return
# Load sections out of the json structure
try:
self._configuration._general = json_structure[ConfigurationReader.GENERAL_SECTION]
except KeyError:
EventLogger.critical("json configuration file has no [" + ConfigurationReader.GENERAL_SECTION + "] section")
self._readConfigErr += 1
self._configuration._xively = prevent_key_error(json_structure, ConfigurationReader.XIVELY_SECTION)
self._configuration._devices = prevent_key_error(json_structure, ConfigurationReader.DEVICES_SECTION)
def create_tree_items(self, blueprint):
"""
Create the device tree with the given blueprint.
Shows all possible devices, if the view_all Flag is True.
"""
self.tree_devices.clear()
self.tree_devices.setSortingEnabled(False)
try:
for dev in blueprint:
self.__add_item_to_tree(dev)
EventLogger.debug("Device Tree created.")
except Exception as e:
EventLogger.warning("DeviceTree - Exception while creating the Tree: " + str(e))
self.tree_devices.sortItems(0, QtCore.Qt.AscendingOrder)
self.tree_devices.setSortingEnabled(True)
def main(config_filename, gui_config, gui_job):
"""
This function initialize the data logger and starts the logging process
"""
config = None
gui_start = False
if config_filename != None: # started via console
config = load_and_validate_config(config_filename)
if config == None:
return None
else: # started via GUI
config = gui_config
gui_start = True
if config['debug']['log']['enabled']:
EventLogger.add_logger(
FileLogger('FileLogger',
log_level_name_to_id(config['debug']['log']['level']),
config['debug']['log']['file_name']))
data_logger = None
try:
data_logger = DataLogger(config, gui_job)
if data_logger.ipcon is not None:
data_logger.run()
if not gui_start:
__exit_condition(data_logger)
else:
raise DataLoggerException(
DataLoggerException.DL_CRITICAL_ERROR,
"DataLogger did not start logging process! Please check for errors."
)
except Exception as exc:
EventLogger.critical(str(exc))
if gui_start:
return None
else:
sys.exit(DataLoggerException.DL_CRITICAL_ERROR)
return data_logger
def create_tree_items(self, blueprint):
"""
Create the device tree with the given blueprint.
Shows all possible devices, if the view_all Flag is True.
"""
self.tree_devices.clear()
self.tree_devices.setSortingEnabled(False)
try:
for dev in blueprint:
self.__add_item_to_tree(dev)
EventLogger.debug("Device Tree created.")
except Exception as e:
EventLogger.warning(
"DeviceTree - Exception while creating the Tree: " + str(e))
self.tree_devices.sortItems(0, QtCore.Qt.AscendingOrder)
self.tree_devices.setSortingEnabled(True)
def _roll_files(self):
i = self._file_count
self.close_file()
while True:
if i == 0:
# first file reached
break
tmp_file_name = Utilities.replace_right(self._file_path, ".", "(" + str(i) + ").", 1)
if os.path.exists(tmp_file_name):
if i == self._file_count:
# max file count -> delete
os.remove(tmp_file_name)
EventLogger.debug("Rolling Files... removed File(" + str(i) + ")")
else:
# copy file and remove old
copyfile(tmp_file_name, Utilities.replace_right(self._file_path, ".", "(" + str(i + 1) + ").", 1))
EventLogger.debug("Rolling Files... copied File(" + str(i) + ") into (" + str(i + 1) + ")")
os.remove(tmp_file_name)
i -= 1
if self._file_count != 0:
copyfile(self._file_path, Utilities.replace_right(self._file_path, ".", "(" + str(1) + ").", 1))
EventLogger.debug("Rolling Files... copied original File into File(1)")
os.remove(self._file_path)
self._open_file_A()
def run(self):
"""
This function starts the actual logging process in a new thread
"""
self.stopped = False
self.process_data_csv_section()
self.initialize_loggable_devices()
"""START-WRITE-THREAD"""
# create jobs
# look which thread should be working
if self.csv_enabled:
self.jobs.append(CSVWriterJob(name="CSV-Writer", datalogger=self))
if self._gui_job is not None:
self._gui_job.set_datalogger(self)
self.jobs.append(self._gui_job)
for t in self.jobs:
t.start()
EventLogger.debug("Jobs started.")
"""START-TIMERS"""
for t in self.timers:
t.start()
EventLogger.debug("Get-Timers started.")
"""END_CONDITIONS"""
EventLogger.info("DataLogger is running...")
def validate(self):
"""
This function performs the validation of the various sections of the JSON
configuration file
"""
EventLogger.info("Validating config file")
self._validate_hosts()
self._validate_data()
self._validate_debug()
self._validate_devices()
if self._error_count > 0:
EventLogger.critical("Validation found {0} errors".format(self._error_count))
else:
EventLogger.info("Validation successful")
#logging_time = self._log_space_counter.calculate_time()
#if self._log_space_counter.file_size != 0:
# EventLogger.info("Logging time until old data will be overwritten.")
# EventLogger.info("Days: " + str(logging_time[0]) +
# " Hours: " + str(logging_time[1]) +
# " Minutes: " + str(logging_time[2]) +
# " Seconds: " + str(logging_time[3]))
#EventLogger.info("Will write about " + str(
# int(self._log_space_counter.lines_per_second + 0.5)) + " lines per second into the log-file.")
return self._error_count == 0
def validate(self):
"""
This function performs the validation of the various sections of the JSON
configuration file
"""
EventLogger.info("Validating config file")
self._validate_hosts()
self._validate_data()
self._validate_debug()
self._validate_devices()
if self._error_count > 0:
EventLogger.critical("Validation found {0} errors".format(self._error_count))
else:
EventLogger.info("Validation successful")
#logging_time = self._log_space_counter.calculate_time()
#if self._log_space_counter.file_size != 0:
# EventLogger.info("Logging time until old data will be overwritten.")
# EventLogger.info("Days: " + str(logging_time[0]) +
# " Hours: " + str(logging_time[1]) +
# " Minutes: " + str(logging_time[2]) +
# " Seconds: " + str(logging_time[3]))
#EventLogger.info("Will write about " + str(
# int(self._log_space_counter.lines_per_second + 0.5)) + " lines per second into the log-file.")
return self._error_count == 0
def __init__(self, config, gui_job):
super(DataLogger, self).__init__()
self.jobs = [] # thread hashmap for all running threads/jobs
self.job_exit_flag = False # flag for stopping the thread
self.job_sleep = 1 # TODO: Enahncement -> use condition objects
self.timers = []
self._gui_job = gui_job
self.data_queue = {} # universal data_queue hash map
self.host = config['hosts']['default']['name']
self.port = config['hosts']['default']['port']
self.loggable_devices = []
self.ipcon = IPConnection()
self.ipcon.register_callback(IPConnection.CALLBACK_CONNECTED, self.cb_connected)
self.ipcon.register_callback(IPConnection.CALLBACK_ENUMERATE, self.cb_enumerate)
try:
self.ipcon.connect(self.host, self.port) # Connect to brickd
except Exception as e:
EventLogger.critical("A critical error occur: " + str(e))
self.ipcon = None
raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e))
EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.")
self.ipcon.set_timeout(1) # TODO: Timeout number
EventLogger.debug("Set ipcon.time_out to 1.")
self._config = config
self.csv_file_name = 'logger_data_{0}.csv'.format(int(time.time()))
self.csv_enabled = True
self.stopped = False
def __init__(self, config, gui_job=None):
"""
config -- brickv.data_logger.configuration_validator.Configuration
"""
super(DataLogger, self).__init__()
self.jobs = [] # thread hashmap for all running threads/jobs
self.job_exit_flag = False # flag for stopping the thread
self.job_sleep = 1 # TODO: Enahncement -> use condition objects
self.timers = []
self._gui_job = gui_job
self.max_file_size = None
self.max_file_count = None
self.data_queue = {} # universal data_queue hash map
self.host = config._general[ConfigurationReader.GENERAL_HOST]
self.port = utils.Utilities.parse_to_int(config._general[ConfigurationReader.GENERAL_PORT])
self.ipcon = IPConnection()
try:
self.ipcon.connect(self.host, self.port) # Connect to brickd
except Exception as e:
EventLogger.critical("A critical error occur: " + str(e))
self.ipcon = None
raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e))
EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.")
self.ipcon.set_timeout(1) # TODO: Timeout number
EventLogger.debug("Set ipcon.time_out to 1.")
self._configuration = config
self.default_file_path = "logged_data.csv"
self.log_to_file = True
self.log_to_xively = False
self.stopped = False
def btn_save_config_clicked(self):
"""
Opens a FileSelectionDialog and saves the current config.
"""
conf = GuiConfigHandler.create_config_file(self)
fn = QtGui.QFileDialog.getSaveFileName(self,
'Save Config-File',
os.getcwd(),
filter='*.json')
if fn == "":
# cancel
EventLogger.debug("Cancelled load Config.")
return
try:
with open(fn, 'w') as outfile:
json.dump(conf, outfile, sort_keys=True, indent=2)
except Exception as e1:
EventLogger.warning("Load Config - Exception: " + str(e1))
QMessageBox.warning(
self, 'Error',
'Could not save the Config-File! Look at the Log-File for further information.',
QMessageBox.Ok)
return
QMessageBox.information(self, 'Success', 'Config-File saved!',
QMessageBox.Ok)
EventLogger.info("Config-File saved to: " + str(fn))
def initialize_loggable_devices(self):
"""
This function creates the actual objects for each device out of the configuration
"""
# start the timers
for device in self._config['devices']:
if len(device['uid']) == 0:
EventLogger.warning('Ignoring "{0}" with empty UID'.format(device['name']))
continue
try:
decoded_uid = base58decode(device['uid'])
except:
EventLogger.warning('Ignoring "{0}" with invalid UID: {1}'.format(device['name'], device['uid']))
continue
if decoded_uid < 1 or decoded_uid > 0xFFFFFFFF:
EventLogger.warning('Ignoring "{0}" with out-of-range UID: {1}'.format(device['name'], device['uid']))
continue
try:
loggable_device = DeviceImpl(device, self)
loggable_device.start_timer()
except Exception as e:
msg = "A critical error occur: " + str(e)
self.stop()
raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, msg)
self.loggable_devices.append(loggable_device)
self.apply_options()
def run(self):
"""
This function starts the actual logging process in a new thread
"""
self.stopped = False
self.process_data_csv_section()
self.initialize_loggable_devices()
"""START-WRITE-THREAD"""
# create jobs
# look which thread should be working
if self.csv_enabled:
self.jobs.append(CSVWriterJob(name="CSV-Writer", datalogger=self))
if self._gui_job is not None:
self._gui_job.set_datalogger(self)
self.jobs.append(self._gui_job)
for t in self.jobs:
t.start()
EventLogger.debug("Jobs started.")
"""START-TIMERS"""
for t in self.timers:
t.start()
EventLogger.debug("Get-Timers started.")
"""END_CONDITIONS"""
EventLogger.info("DataLogger is running...")
def btn_remove_device_clicked(self):
"""
Removes selected Device
"""
selected_item = self.tree_devices.selectedItems()
for index in range(0, len(selected_item)):
try:
if selected_item[index] is None:
continue
device_name = selected_item[index].text(0)
device_id = selected_item[index].text(1)
if selected_item[index].text(
0) not in Identifier.DEVICE_DEFINITIONS:
# have to find the parent
current_item = selected_item[0]
while True:
if current_item.parent() is None:
if current_item.text(
0) not in Identifier.DEVICE_DEFINITIONS:
EventLogger.error("Cant remove device: " +
selected_item[index].text(0))
device_name = ""
device_id = ""
break
else:
device_name = current_item.text(0)
device_id = current_item.text(1)
break
else:
current_item = current_item.parent()
self.remove_item_from_tree(device_name, device_id)
except Exception as e:
if not str(e).startswith("wrapped C/C++ object"):
EventLogger.error("Cant remove device: " +
str(e)) # was already removed
def __init__(self, config, gui_job):
super(DataLogger, self).__init__()
self.daemon = True
self.jobs = [] # thread hashmap for all running threads/jobs
self.job_exit_flag = False # flag for stopping the thread
self.job_sleep = 1 # TODO: Enahncement -> use condition objects
self.timers = []
self._gui_job = gui_job
self.data_queue = {} # universal data_queue hash map
self.host = config['hosts']['default']['name']
self.port = config['hosts']['default']['port']
self.secret = config['hosts']['default']['secret']
if self.secret != None:
try:
self.secret = self.secret.encode('ascii')
except:
EventLogger.critical(
'Authentication secret cannot contain non-ASCII characters'
)
self.secret = None
self.loggable_devices = []
self.ipcon = IPConnection()
self.ipcon.register_callback(IPConnection.CALLBACK_CONNECTED,
self.cb_connected)
self.ipcon.register_callback(IPConnection.CALLBACK_ENUMERATE,
self.cb_enumerate)
try:
self.ipcon.connect(self.host, self.port) # Connect to brickd
except Exception as e:
EventLogger.critical("A critical error occur: " + str(e))
self.ipcon = None
raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR,
"A critical error occur: " + str(e))
EventLogger.info("Connection to " + self.host + ":" + str(self.port) +
" established.")
self.ipcon.set_timeout(1) # TODO: Timeout number
EventLogger.debug("Set ipcon.time_out to 1.")
self._config = config
self.csv_file_name = 'logger_data_{0}.csv'.format(int(time.time()))
self.csv_enabled = True
self.stopped = False
def _read_json_config_file(self):
with codecs.open(self.fileName, 'r', 'UTF-8') as content_file:
try:
json_structure = json.load(content_file)
except ValueError as e:
EventLogger.critical("Cant parse the configuration file: " +
str(e))
return
# Load sections out of the json structure
try:
self._configuration._general = json_structure[
ConfigurationReader.GENERAL_SECTION]
except KeyError:
EventLogger.critical("json configuration file has no [" +
ConfigurationReader.GENERAL_SECTION +
"] section")
self._readConfigErr += 1
self._configuration._xively = prevent_key_error(
json_structure, ConfigurationReader.XIVELY_SECTION)
self._configuration._devices = prevent_key_error(
json_structure, ConfigurationReader.DEVICES_SECTION)