def _job(self): try: # check for datalogger object if AbstractJob._job(self): return EventLogger.debug(self._job_name + " Started") while True: if not self._datalogger.data_queue[self.name].empty(): csv_data = self._get_data_from_queue() self.signalNewData.emit(csv_data) if not self._exit_flag and self._datalogger.data_queue[ self.name].empty(): time.sleep(self._datalogger.job_sleep) if self._exit_flag and self._datalogger.data_queue[ self.name].empty(): self._remove_from_data_queue() break except Exception as e: EventLogger.critical(self._job_name + " -.- " + str(e)) self.stop()
def _btn_add_device_clicked(self): """ Add the selected device from the list/DataLogger-Config. """ items = self.list_widget.selectedItems() cur_dev = GuiConfigHandler.get_simple_blueprint(self.Ui_Logger) for item in items: name = item.text() if name == self._no_connected_device_string or name == self._list_separator_string: # ignore those continue dev_name, uid = Utilities.parse_device_name(name) dev = GuiConfigHandler.get_device_blueprint(dev_name) if dev is None: EventLogger.debug("DeviceDialog._btn_add_device_clicked: Blueprint(" + str(dev_name) + ") was None!") continue if uid is not None: if self.__is_device_in_list(dev_name, uid, cur_dev): continue # else dev[Identifier.DD_UID] = uid else: dev[Identifier.DD_UID] = Identifier.DD_UID_DEFAULT self._logger_window.add_item_to_tree(dev)
def __init__(self, config, gui_job=None): """ config -- brickv.data_logger.configuration_validator.Configuration """ super(DataLogger, self).__init__() self.jobs = [] # thread hashmap for all running threads/jobs self.job_exit_flag = False # flag for stopping the thread self.job_sleep = 1 # TODO: Enahncement -> use condition objects self.timers = [] self._gui_job = gui_job self.max_file_size = None self.max_file_count = None self.data_queue = {} # universal data_queue hash map self.host = config._general[ConfigurationReader.GENERAL_HOST] self.port = utils.Utilities.parse_to_int(config._general[ConfigurationReader.GENERAL_PORT]) self.ipcon = IPConnection() try: self.ipcon.connect(self.host, self.port) # Connect to brickd except Exception as e: EventLogger.critical("A critical error occur: " + str(e)) self.ipcon = None raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e)) EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.") self.ipcon.set_timeout(1) # TODO: Timeout number EventLogger.debug("Set ipcon.time_out to 1.") self._configuration = config self.default_file_path = "logged_data.csv" self.log_to_file = True self.log_to_xively = False self.stopped = False
def _job(self): try: # check for datalogger object if AbstractJob._job(self): return EventLogger.debug(self._job_name + " Started") csv_writer = CSVWriter(self._datalogger.default_file_path, self._datalogger.max_file_size, self._datalogger.max_file_count) while True: if not self._datalogger.data_queue[self.name].empty(): csv_data = self._get_data_from_queue() EventLogger.debug(self._job_name + " -> " + str(csv_data)) if not csv_writer.write_data_row(csv_data): EventLogger.warning(self._job_name + " Could not write csv row!") if not self._exit_flag and self._datalogger.data_queue[self.name].empty(): time.sleep(self._datalogger.job_sleep) if self._exit_flag and self._datalogger.data_queue[self.name].empty(): exit_return_value = csv_writer.close_file() if exit_return_value: EventLogger.debug(self._job_name + " Closed his csv_writer") else: EventLogger.debug( self._job_name + " Could NOT close his csv_writer! EXIT_RETURN_VALUE=" + str(exit)) EventLogger.debug(self._job_name + " Finished") self._remove_from_data_queue() break except Exception as e: EventLogger.critical(self._job_name + " " + str(e)) self.stop()
def update_setup_tab(self, config): EventLogger.debug('Updating setup tab from config') name = config['hosts']['default']['name'] port = config['hosts']['default']['port'] secret = config['hosts']['default']['secret'] i = self.combo_host.findText(name) if i >= 0: self.combo_host.setCurrentIndex(i) else: self.combo_host.insertItem(0, name, (port, secret != None, secret)) self.combo_host.setCurrentIndex(0) self.spin_port.setValue(port) self.check_authentication.setChecked(secret != None) self.edit_secret.setText(secret if secret != None else '') self.combo_data_time_format.setCurrentIndex(max(self.combo_data_time_format.findData(config['data']['time_format']), 0)) self.edit_data_time_format_strftime.setText(config['data']['time_format_strftime']) self.check_data_to_csv_file.setChecked(config['data']['csv']['enabled']) self.edit_csv_file_name.setText(config['data']['csv']['file_name']) self.combo_debug_time_format.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['time_format']), 0)) self.check_debug_to_log_file.setChecked(config['debug']['log']['enabled']) self.edit_log_file_name.setText(config['debug']['log']['file_name']) self.combo_log_level.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['log']['level']), 0))
def update_devices_tab(self, config): EventLogger.debug('Updating devices tab from config') self.model_devices.removeRows(0, self.model_data.rowCount()) for device in config['devices']: self.add_device_to_tree(device)
def run(self): """ This function starts the actual logging process in a new thread """ self.stopped = False self.process_data_csv_section() self.initialize_loggable_devices() """START-WRITE-THREAD""" # create jobs # look which thread should be working if self.csv_enabled: self.jobs.append(CSVWriterJob(name="CSV-Writer", datalogger=self)) if self._gui_job is not None: self._gui_job.set_datalogger(self) self.jobs.append(self._gui_job) for t in self.jobs: t.start() EventLogger.debug("Jobs started.") """START-TIMERS""" for t in self.timers: t.start() EventLogger.debug("Get-Timers started.") """END_CONDITIONS""" EventLogger.info("DataLogger is running...")
def update_setup_tab(self, config): EventLogger.debug('Updating setup tab from config') name = config['hosts']['default']['name'] port = config['hosts']['default']['port'] secret = config['hosts']['default']['secret'] i = self.combo_host.findText(name) if i >= 0: self.combo_host.setCurrentIndex(i) else: self.combo_host.insertItem(0, name, (port, secret != None, secret)) self.combo_host.setCurrentIndex(0) self.spin_port.setValue(port) self.check_authentication.setChecked(secret != None) self.edit_secret.setText(secret if secret != None else '') self.combo_data_time_format.setCurrentIndex(max(self.combo_data_time_format.findData(config['data']['time_format']), 0)) self.edit_data_time_format_strftime.setText(config['data']['time_format_strftime']) self.check_data_to_csv_file.setChecked(config['data']['csv']['enabled']) self.edit_csv_file_name.setText(config['data']['csv']['file_name']) self.combo_debug_time_format.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['time_format']), 0)) self.check_debug_to_log_file.setChecked(config['debug']['log']['enabled']) self.edit_log_file_name.setText(config['debug']['log']['file_name']) self.combo_log_level.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['log']['level']), 0))
def run(self): """ This function starts the actual logging process in a new thread """ self.stopped = False self.process_data_csv_section() self.initialize_loggable_devices() """START-WRITE-THREAD""" # create jobs # look which thread should be working if self.csv_enabled: self.jobs.append(CSVWriterJob(name="CSV-Writer", datalogger=self)) if self._gui_job is not None: self._gui_job.set_datalogger(self) self.jobs.append(self._gui_job) for t in self.jobs: t.start() EventLogger.debug("Jobs started.") """START-TIMERS""" for t in self.timers: t.start() EventLogger.debug("Get-Timers started.") """END_CONDITIONS""" EventLogger.info("DataLogger is running...")
def btn_load_config_clicked(self): """ Opens a FileSelectionDialog and loads the selected config. """ fn = QtGui.QFileDialog.getOpenFileName(self, "Open Config-File...", os.getcwd(), "JSON-Files (*.json)") if fn == "": # cancel EventLogger.debug("Cancelled save Config.") return config_json = None try: with codecs.open(fn, 'r', 'UTF-8') as content_file: try: config_json = json.load(content_file) except ValueError as e: EventLogger.warning("Load Config - Cant parse the configuration file: " + str(e)) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) return EventLogger.info("Loaded Config-File from: " + str(fn)) # devices config_blueprint = GuiConfigHandler.load_devices(config_json) if config_blueprint is None: return self.create_tree_items(config_blueprint) # general_section from brickv.data_logger.configuration_validator import ConfigurationReader self.update_setup_tab(config_json[ConfigurationReader.GENERAL_SECTION])
def update_devices_tab(self, config): EventLogger.debug('Updating devices tab from config') self.model_devices.removeRows(0, self.model_data.rowCount()) for device in config['devices']: self.add_device_to_tree(device)
def stop(self): """ This function ends the logging process. self.stopped will be set to True if the data logger stops """ EventLogger.info("Closing Timers and Threads...") """CLEANUP_AFTER_STOP """ # check if all timers stopped for t in self.timers: t.stop() for t in self.timers: t.join() EventLogger.debug("Get-Timers[" + str(len(self.timers)) + "] stopped.") # set THREAD_EXIT_FLAG for all work threads for job in self.jobs: job.stop() # wait for all threads to stop for job in self.jobs: job.join() EventLogger.debug("Jobs[" + str(len(self.jobs)) + "] stopped.") try: self.ipcon.disconnect() except: pass EventLogger.info("Connection closed successfully.") self.stopped = True
def update_setup_tab(self, config): EventLogger.debug('Updating setup tab from config') self.combo_host.setEditText(config['hosts']['default']['name']) self.spin_port.setValue(config['hosts']['default']['port']) self.combo_data_time_format.setCurrentIndex( max( self.combo_data_time_format.findData( config['data']['time_format']), 0)) self.check_data_to_csv_file.setChecked( config['data']['csv']['enabled']) self.edit_csv_file_name.setText( config['data']['csv']['file_name'].decode('utf-8')) self.combo_debug_time_format.setCurrentIndex( max( self.combo_debug_time_format.findData( config['debug']['time_format']), 0)) self.check_debug_to_log_file.setChecked( config['debug']['log']['enabled']) self.edit_log_file_name.setText( config['debug']['log']['file_name'].decode('utf-8')) self.combo_log_level.setCurrentIndex( max( self.combo_debug_time_format.findData( config['debug']['log']['level']), 0))
def __init__(self, config, gui_job): super(DataLogger, self).__init__() self.jobs = [] # thread hashmap for all running threads/jobs self.job_exit_flag = False # flag for stopping the thread self.job_sleep = 1 # TODO: Enahncement -> use condition objects self.timers = [] self._gui_job = gui_job self.data_queue = {} # universal data_queue hash map self.host = config['hosts']['default']['name'] self.port = config['hosts']['default']['port'] self.loggable_devices = [] self.ipcon = IPConnection() self.ipcon.register_callback(IPConnection.CALLBACK_CONNECTED, self.cb_connected) self.ipcon.register_callback(IPConnection.CALLBACK_ENUMERATE, self.cb_enumerate) try: self.ipcon.connect(self.host, self.port) # Connect to brickd except Exception as e: EventLogger.critical("A critical error occur: " + str(e)) self.ipcon = None raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e)) EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.") self.ipcon.set_timeout(1) # TODO: Timeout number EventLogger.debug("Set ipcon.time_out to 1.") self._config = config self.csv_file_name = 'logger_data_{0}.csv'.format(int(time.time())) self.csv_enabled = True self.stopped = False
def btn_save_config_clicked(self): """ Opens a FileSelectionDialog and saves the current config. """ conf = GuiConfigHandler.create_config_file(self) fn = QtGui.QFileDialog.getSaveFileName(self, 'Save Config-File', os.getcwd(), filter='*.json') if fn == "": # cancel EventLogger.debug("Cancelled load Config.") return try: with open(fn, 'w') as outfile: json.dump(conf, outfile, sort_keys=True, indent=2) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) QMessageBox.warning( self, 'Error', 'Could not save the Config-File! Look at the Log-File for further information.', QMessageBox.Ok) return QMessageBox.information(self, 'Success', 'Config-File saved!', QMessageBox.Ok) EventLogger.info("Config-File saved to: " + str(fn))
def stop(self): """ This function ends the logging process. self.stopped will be set to True if the data logger stops """ EventLogger.info("Closing Timers and Threads...") """CLEANUP_AFTER_STOP """ # check if all timers stopped for t in self.timers: t.stop() for t in self.timers: t.join() EventLogger.debug("Get-Timers[" + str(len(self.timers)) + "] stopped.") # set THREAD_EXIT_FLAG for all work threads for job in self.jobs: job.stop() # wait for all threads to stop for job in self.jobs: job.join() EventLogger.debug("Jobs[" + str(len(self.jobs)) + "] stopped.") try: self.ipcon.disconnect() except: pass EventLogger.info("Connection closed successfully.") self.stopped = True
def _write_header(self): """Writes a csv header into the file""" if not self._file_is_empty(): EventLogger.debug("File is not empty") return EventLogger.debug("CSVWriter._write_header() - done") self._csv_file.writerow(["TIME"] + ["NAME"] + ["UID"] + ["VAR"] + ["RAW"] + ["UNIT"]) self._raw_file.flush()
def process_data_csv_section(self): """ Information out of the general section will be consumed here """ csv = self._config['data']['csv'] self.csv_enabled = csv['enabled'] self.csv_file_name = csv['file_name'] EventLogger.debug("Logging output to CSV file: " + str(self.csv_enabled)) EventLogger.debug("Output file path: " + str(self.csv_file_name))
def btn_set_eventfile_clicked(self): """ Opens a FileSelectionDialog and sets the selected path for the event output file. """ fn = self.__choose_file_dialog('Choose Eventfile destination', "Log-Files (*.log)") if fn == "": # cancel EventLogger.debug("Cancelled select Eventfile-Path.") return self.line_path_to_eventfile.setText(fn)
def btn_set_eventfile_clicked(self): """ Opens a FileSelectionDialog and sets the selected path for the event output file. """ fn = self.__choose_file_dialog('Choose Eventfile destination', "Log-Files (*.log)") if fn == "": # cancel EventLogger.debug("Cancelled select Eventfile-Path.") return self.line_path_to_eventfile.setText(fn)
def btn_set_logfile_clicked(self): """ Opens a FileSelectionDialog and sets the selected path for the data output file. """ fn = self.__choose_file_dialog('Choose Config Destination', "CSV-Files (*.csv);;Text-Files (*.txt)") if fn == "": # cancel self.line_path_to_file.setText("") EventLogger.debug("Cancelled select Config-File-Path.") return self.line_path_to_file.setText(fn)
def process_general_section(self): """ Information out of the general section will be consumed here """ data = self._configuration._general self.log_to_file = data[ConfigurationReader.GENERAL_LOG_TO_FILE] self.default_file_path = data[ConfigurationReader.GENERAL_PATH_TO_FILE] self.max_file_size = data[ConfigurationReader.GENERAL_LOG_FILE_SIZE] self.max_file_count = data[ConfigurationReader.GENERAL_LOG_COUNT] EventLogger.debug("Logging output to file: " + str(self.log_to_file)) EventLogger.debug("Output file path: " + str(self.default_file_path))
def _job(self): try: # check for datalogger object if AbstractJob._job(self): return EventLogger.debug(self._job_name + " Started") csv_writer = CSVWriter(self._datalogger.csv_file_name) while True: if not self._datalogger.data_queue[self.name].empty(): csv_data = self._get_data_from_queue() #EventLogger.debug(self._job_name + " -> " + str(csv_data)) if not csv_writer.write_data_row(csv_data): EventLogger.warning(self._job_name + " Could not write csv row!") if not self._exit_flag and self._datalogger.data_queue[self.name].empty(): time.sleep(self._datalogger.job_sleep) if self._exit_flag and self._datalogger.data_queue[self.name].empty(): exit_return_value = csv_writer.close_file() if exit_return_value: EventLogger.debug(self._job_name + " Closed his csv_writer") else: EventLogger.debug( self._job_name + " Could NOT close his csv_writer! EXIT_RETURN_VALUE=" + str(exit)) EventLogger.debug(self._job_name + " Finished") self._remove_from_data_queue() break except Exception as e: EventLogger.critical(self._job_name + " " + str(e)) self.stop()
def update_setup_tab(self, config): EventLogger.debug('Updating setup tab from config') self.combo_host.setEditText(config['hosts']['default']['name']) self.spin_port.setValue(config['hosts']['default']['port']) self.combo_data_time_format.setCurrentIndex(max(self.combo_data_time_format.findData(config['data']['time_format']), 0)) self.check_data_to_csv_file.setChecked(config['data']['csv']['enabled']) self.edit_csv_file_name.setText(config['data']['csv']['file_name'].decode('utf-8')) self.combo_debug_time_format.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['time_format']), 0)) self.check_debug_to_log_file.setChecked(config['debug']['log']['enabled']) self.edit_log_file_name.setText(config['debug']['log']['file_name'].decode('utf-8')) self.combo_log_level.setCurrentIndex(max(self.combo_debug_time_format.findData(config['debug']['log']['level']), 0))
def btn_set_logfile_clicked(self): """ Opens a FileSelectionDialog and sets the selected path for the data output file. """ fn = self.__choose_file_dialog( 'Choose Config Destination', "CSV-Files (*.csv);;Text-Files (*.txt)") if fn == "": # cancel self.line_path_to_file.setText("") EventLogger.debug("Cancelled select Config-File-Path.") return self.line_path_to_file.setText(fn)
def __init__(self, config, gui_job): super(DataLogger, self).__init__() self.daemon = True self.jobs = [] # thread hashmap for all running threads/jobs self.job_exit_flag = False # flag for stopping the thread self.job_sleep = 1 # TODO: Enahncement -> use condition objects self.timers = [] self._gui_job = gui_job self.data_queue = {} # universal data_queue hash map self.host = config['hosts']['default']['name'] self.port = config['hosts']['default']['port'] self.secret = config['hosts']['default']['secret'] if self.secret != None: try: self.secret = self.secret.encode('ascii') except: EventLogger.critical( 'Authentication secret cannot contain non-ASCII characters' ) self.secret = None self.loggable_devices = [] self.ipcon = IPConnection() self.ipcon.register_callback(IPConnection.CALLBACK_CONNECTED, self.cb_connected) self.ipcon.register_callback(IPConnection.CALLBACK_ENUMERATE, self.cb_enumerate) try: self.ipcon.connect(self.host, self.port) # Connect to brickd except Exception as e: EventLogger.critical("A critical error occur: " + str(e)) self.ipcon = None raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e)) EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.") self.ipcon.set_timeout(1) # TODO: Timeout number EventLogger.debug("Set ipcon.time_out to 1.") self._config = config self.csv_file_name = 'logger_data_{0}.csv'.format(int(time.time())) self.csv_enabled = True self.stopped = False
def create_tree_items(self, blueprint): """ Create the device tree with the given blueprint. Shows all possible devices, if the view_all Flag is True. """ self.tree_devices.clear() self.tree_devices.setSortingEnabled(False) try: for dev in blueprint: self.__add_item_to_tree(dev) EventLogger.debug("Device Tree created.") except Exception as e: EventLogger.warning("DeviceTree - Exception while creating the Tree: " + str(e)) self.tree_devices.sortItems(0, QtCore.Qt.AscendingOrder) self.tree_devices.setSortingEnabled(True)
def create_tree_items(self, blueprint): """ Create the device tree with the given blueprint. Shows all possible devices, if the view_all Flag is True. """ self.tree_devices.clear() self.tree_devices.setSortingEnabled(False) try: for dev in blueprint: self.__add_item_to_tree(dev) EventLogger.debug("Device Tree created.") except Exception as e: EventLogger.warning( "DeviceTree - Exception while creating the Tree: " + str(e)) self.tree_devices.sortItems(0, QtCore.Qt.AscendingOrder) self.tree_devices.setSortingEnabled(True)
def _roll_files(self): i = self._file_count self.close_file() while True: if i == 0: # first file reached break tmp_file_name = Utilities.replace_right(self._file_path, ".", "(" + str(i) + ").", 1) if os.path.exists(tmp_file_name): if i == self._file_count: # max file count -> delete os.remove(tmp_file_name) EventLogger.debug("Rolling Files... removed File(" + str(i) + ")") else: # copy file and remove old copyfile(tmp_file_name, Utilities.replace_right(self._file_path, ".", "(" + str(i + 1) + ").", 1)) EventLogger.debug("Rolling Files... copied File(" + str(i) + ") into (" + str(i + 1) + ")") os.remove(tmp_file_name) i -= 1 if self._file_count != 0: copyfile(self._file_path, Utilities.replace_right(self._file_path, ".", "(" + str(1) + ").", 1)) EventLogger.debug("Rolling Files... copied original File into File(1)") os.remove(self._file_path) self._open_file_A()
def btn_load_config_clicked(self): """ Opens a FileSelectionDialog and loads the selected config. """ fn = QtGui.QFileDialog.getOpenFileName(self, "Open Config-File...", os.getcwd(), "JSON-Files (*.json)") if fn == "": # cancel EventLogger.debug("Cancelled save Config.") return config_json = None try: with codecs.open(fn, 'r', 'UTF-8') as content_file: try: config_json = json.load(content_file) except ValueError as e: EventLogger.warning( "Load Config - Cant parse the configuration file: " + str(e)) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) return EventLogger.info("Loaded Config-File from: " + str(fn)) # devices config_blueprint = GuiConfigHandler.load_devices(config_json) if config_blueprint is None: return self.create_tree_items(config_blueprint) # general_section from brickv.data_logger.configuration_validator import ConfigurationReader self.update_setup_tab(config_json[ConfigurationReader.GENERAL_SECTION])
def _job(self): try: # check for datalogger object if AbstractJob._job(self): return EventLogger.debug(self._job_name + " Started") while True: if not self._datalogger.data_queue[self.name].empty(): csv_data = self._get_data_from_queue() self.signalNewData.emit(csv_data) if not self._exit_flag and self._datalogger.data_queue[self.name].empty(): time.sleep(self._datalogger.job_sleep) if self._exit_flag and self._datalogger.data_queue[self.name].empty(): self._remove_from_data_queue() break except Exception as e: EventLogger.critical(self._job_name + " -.- " + str(e)) self.stop()
def btn_save_config_clicked(self): """ Opens a FileSelectionDialog and saves the current config. """ conf = GuiConfigHandler.create_config_file(self) fn = QtGui.QFileDialog.getSaveFileName(self, 'Save Config-File', os.getcwd(), filter='*.json') if fn == "": # cancel EventLogger.debug("Cancelled load Config.") return try: with open(fn, 'w') as outfile: json.dump(conf, outfile, sort_keys=True, indent=2) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) QMessageBox.warning(self, 'Error', 'Could not save the Config-File! Look at the Log-File for further information.', QMessageBox.Ok) return QMessageBox.information(self, 'Success', 'Config-File saved!', QMessageBox.Ok) EventLogger.info("Config-File saved to: " + str(fn))
def start_timer(self): """ Starts all timer for all loggable variables of the devices. """ EventLogger.debug(self.__str__())