def _rolling_file(self): f_size = os.path.getsize(self._file_path) if f_size > self._file_size: # self.set_file_path(self._create_new_file_name(self._file_path)) EventLogger.info( "Max Filesize(" + "%.3f" % (self._file_size / 1024.0 / 1024.0) + " MB) reached! Rolling Files...") self._roll_files()
def btn_save_config_clicked(self): """ Opens a FileSelectionDialog and saves the current config. """ conf = GuiConfigHandler.create_config_file(self) fn = QtGui.QFileDialog.getSaveFileName(self, 'Save Config-File', os.getcwd(), filter='*.json') if fn == "": # cancel EventLogger.debug("Cancelled load Config.") return try: with open(fn, 'w') as outfile: json.dump(conf, outfile, sort_keys=True, indent=2) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) QMessageBox.warning( self, 'Error', 'Could not save the Config-File! Look at the Log-File for further information.', QMessageBox.Ok) return QMessageBox.information(self, 'Success', 'Config-File saved!', QMessageBox.Ok) EventLogger.info("Config-File saved to: " + str(fn))
def run(self): """ This function starts the actual logging process in a new thread """ self.stopped = False self.process_data_csv_section() self.initialize_loggable_devices() """START-WRITE-THREAD""" # create jobs # look which thread should be working if self.csv_enabled: self.jobs.append(CSVWriterJob(name="CSV-Writer", datalogger=self)) if self._gui_job is not None: self._gui_job.set_datalogger(self) self.jobs.append(self._gui_job) for t in self.jobs: t.start() EventLogger.debug("Jobs started.") """START-TIMERS""" for t in self.timers: t.start() EventLogger.debug("Get-Timers started.") """END_CONDITIONS""" EventLogger.info("DataLogger is running...")
def __init__(self, config, gui_job): super(DataLogger, self).__init__() self.jobs = [] # thread hashmap for all running threads/jobs self.job_exit_flag = False # flag for stopping the thread self.job_sleep = 1 # TODO: Enahncement -> use condition objects self.timers = [] self._gui_job = gui_job self.data_queue = {} # universal data_queue hash map self.host = config['hosts']['default']['name'] self.port = config['hosts']['default']['port'] self.loggable_devices = [] self.ipcon = IPConnection() self.ipcon.register_callback(IPConnection.CALLBACK_CONNECTED, self.cb_connected) self.ipcon.register_callback(IPConnection.CALLBACK_ENUMERATE, self.cb_enumerate) try: self.ipcon.connect(self.host, self.port) # Connect to brickd except Exception as e: EventLogger.critical("A critical error occur: " + str(e)) self.ipcon = None raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e)) EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.") self.ipcon.set_timeout(1) # TODO: Timeout number EventLogger.debug("Set ipcon.time_out to 1.") self._config = config self.csv_file_name = 'logger_data_{0}.csv'.format(int(time.time())) self.csv_enabled = True self.stopped = False
def _rolling_file(self): f_size = os.path.getsize(self._file_path) if f_size > self._file_size: EventLogger.info("Max Filesize(" + "%.3f" % (self._file_size / 1024.0 / 1024.0) + " MB) reached! Rolling Files...") self._roll_files()
def signal_handler(interrupted_ref, signum, frame): """ This function handles the ctrl + c exit condition if it's raised through the console """ EventLogger.info('Received SIGINT/SIGTERM') interrupted_ref[0] = True
def validate(self): """ This function performs the validation of the various sections of the json configuration file """ EventLogger.info("Started configuration file validation") self.validate_general_section() self.validate_xively_section() self.validate_devices_section() EventLogger.info("Validation ends with [" + str(self._error_count) + "] errors") logging_time = self._log_space_counter.calculate_time() if self._log_space_counter.file_size != 0: EventLogger.info( "Logging time until old data will be overwritten.") EventLogger.info("Days: " + str(logging_time[0]) + " Hours: " + str(logging_time[1]) + " Minutes: " + str(logging_time[2]) + " Seconds: " + str(logging_time[3])) EventLogger.info( "Will write about " + str(int(self._log_space_counter.lines_per_second + 0.5)) + " lines per second into the log-file.") if self._error_count != 0: raise DataLoggerException(DataLoggerException.DL_FAILED_VALIDATION, "Validation process found some errors")
def validate(self): """ This function performs the validation of the various sections of the JSON configuration file """ EventLogger.info("Validating config file") self._validate_hosts() self._validate_data() self._validate_debug() self._validate_devices() if self._error_count > 0: EventLogger.critical("Validation found {0} errors".format(self._error_count)) else: EventLogger.info("Validation successful") #logging_time = self._log_space_counter.calculate_time() #if self._log_space_counter.file_size != 0: # EventLogger.info("Logging time until old data will be overwritten.") # EventLogger.info("Days: " + str(logging_time[0]) + # " Hours: " + str(logging_time[1]) + # " Minutes: " + str(logging_time[2]) + # " Seconds: " + str(logging_time[3])) #EventLogger.info("Will write about " + str( # int(self._log_space_counter.lines_per_second + 0.5)) + " lines per second into the log-file.") return self._error_count == 0
def stop(self): """ This function ends the logging process. self.stopped will be set to True if the data logger stops """ EventLogger.info("Closing Timers and Threads...") """CLEANUP_AFTER_STOP """ # check if all timers stopped for t in self.timers: t.stop() for t in self.timers: t.join() EventLogger.debug("Get-Timers[" + str(len(self.timers)) + "] stopped.") # set THREAD_EXIT_FLAG for all work threads for job in self.jobs: job.stop() # wait for all threads to stop for job in self.jobs: job.join() EventLogger.debug("Jobs[" + str(len(self.jobs)) + "] stopped.") try: self.ipcon.disconnect() except: pass EventLogger.info("Connection closed successfully.") self.stopped = True
def __init__(self, config, gui_job=None): """ config -- brickv.data_logger.configuration_validator.Configuration """ super(DataLogger, self).__init__() self.jobs = [] # thread hashmap for all running threads/jobs self.job_exit_flag = False # flag for stopping the thread self.job_sleep = 1 # TODO: Enahncement -> use condition objects self.timers = [] self._gui_job = gui_job self.max_file_size = None self.max_file_count = None self.data_queue = {} # universal data_queue hash map self.host = config._general[ConfigurationReader.GENERAL_HOST] self.port = utils.Utilities.parse_to_int(config._general[ConfigurationReader.GENERAL_PORT]) self.ipcon = IPConnection() try: self.ipcon.connect(self.host, self.port) # Connect to brickd except Exception as e: EventLogger.critical("A critical error occur: " + str(e)) self.ipcon = None raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e)) EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.") self.ipcon.set_timeout(1) # TODO: Timeout number EventLogger.debug("Set ipcon.time_out to 1.") self._configuration = config self.default_file_path = "logged_data.csv" self.log_to_file = True self.log_to_xively = False self.stopped = False
def btn_load_config_clicked(self): """ Opens a FileSelectionDialog and loads the selected config. """ fn = QtGui.QFileDialog.getOpenFileName(self, "Open Config-File...", os.getcwd(), "JSON-Files (*.json)") if fn == "": # cancel EventLogger.debug("Cancelled save Config.") return config_json = None try: with codecs.open(fn, 'r', 'UTF-8') as content_file: try: config_json = json.load(content_file) except ValueError as e: EventLogger.warning("Load Config - Cant parse the configuration file: " + str(e)) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) return EventLogger.info("Loaded Config-File from: " + str(fn)) # devices config_blueprint = GuiConfigHandler.load_devices(config_json) if config_blueprint is None: return self.create_tree_items(config_blueprint) # general_section from brickv.data_logger.configuration_validator import ConfigurationReader self.update_setup_tab(config_json[ConfigurationReader.GENERAL_SECTION])
def process_data_csv_section(self): """ Information out of the general section will be consumed here """ csv = self._config['data']['csv'] self.csv_enabled = csv['enabled'] self.csv_file_name = csv['file_name'] if self.csv_enabled: EventLogger.info("Logging data to CSV file: " + str(self.csv_file_name))
def save_config(config, filename): EventLogger.info('Saving config to file: {0}'.format(filename)) try: s = json.dumps(config, ensure_ascii=False, sort_keys=True, indent=2).encode('utf-8') with open(filename, 'wb') as f: f.write(s) except Exception as e: EventLogger.critical('Could not write config file as JSON: {0}'.format(e)) return False EventLogger.info('Config successfully saved to: {0}'.format(filename)) return True
def __init__(self, config, gui_job): super(DataLogger, self).__init__() self.daemon = True self.jobs = [] # thread hashmap for all running threads/jobs self.job_exit_flag = False # flag for stopping the thread self.job_sleep = 1 # TODO: Enahncement -> use condition objects self.timers = [] self._gui_job = gui_job self.data_queue = {} # universal data_queue hash map self.host = config['hosts']['default']['name'] self.port = config['hosts']['default']['port'] self.secret = config['hosts']['default']['secret'] if self.secret != None: try: self.secret = self.secret.encode('ascii') except: EventLogger.critical( 'Authentication secret cannot contain non-ASCII characters' ) self.secret = None self.loggable_devices = [] self.ipcon = IPConnection() self.ipcon.register_callback(IPConnection.CALLBACK_CONNECTED, self.cb_connected) self.ipcon.register_callback(IPConnection.CALLBACK_ENUMERATE, self.cb_enumerate) try: self.ipcon.connect(self.host, self.port) # Connect to brickd except Exception as e: EventLogger.critical("A critical error occur: " + str(e)) self.ipcon = None raise DataLoggerException(DataLoggerException.DL_CRITICAL_ERROR, "A critical error occur: " + str(e)) EventLogger.info("Connection to " + self.host + ":" + str(self.port) + " established.") self.ipcon.set_timeout(1) # TODO: Timeout number EventLogger.debug("Set ipcon.time_out to 1.") self._config = config self.csv_file_name = 'logger_data_{0}.csv'.format(int(time.time())) self.csv_enabled = True self.stopped = False
def validate(self): """ This function performs the validation of the various sections of the JSON configuration file """ EventLogger.info("Validating config file") self._validate_hosts() self._validate_data() self._validate_debug() self._validate_devices() if self._error_count > 0: EventLogger.critical("Validation found {0} errors".format(self._error_count)) else: EventLogger.info("Validation successful") return self._error_count == 0
def load_and_validate_config(filename): EventLogger.info('Loading config from file: {0}'.format(filename)) try: with open(filename, 'r') as f: s = f.read() config = json.loads(s) except Exception as e: EventLogger.critical('Could not parse config file as JSON: {0}'.format(e)) return None if not ConfigValidator(config).validate(): return None EventLogger.info('Config successfully loaded from: {0}'.format(filename)) return config
def load_and_validate_config(filename): EventLogger.info('Loading config from file: {0}'.format(filename)) try: with open(filename, 'r') as f: s = f.read() config = json.loads(s) except Exception as e: EventLogger.critical( 'Could not parse config file as JSON: {0}'.format(e)) return None if not ConfigValidator(config).validate(): return None EventLogger.info('Config successfully loaded from: {0}'.format(filename)) return config
def validate(self): """ This function performs the validation of the various sections of the JSON configuration file """ EventLogger.info("Validating config file") self._validate_hosts() self._validate_data() self._validate_debug() self._validate_devices() if self._error_count > 0: EventLogger.critical("Validation found {0} errors".format( self._error_count)) else: EventLogger.info("Validation successful") return self._error_count == 0
def cb_connected(self, connect_reason): if self.secret != None: try: secret = self.secret.encode('ascii') except: try: self.ipcon.disconnect() except: pass EventLogger.critical( 'Authentication secret cannot contain non-ASCII characters' ) return self.ipcon.set_auto_reconnect( False) # don't auto-reconnect on authentication error try: self.ipcon.authenticate(secret) except: try: self.ipcon.disconnect() except: pass if connect_reason == IPConnection.CONNECT_REASON_AUTO_RECONNECT: extra = ' after auto-reconnect' else: extra = '' EventLogger.critical('Could not authenticate' + extra) return self.ipcon.set_auto_reconnect(True) EventLogger.info("Successfully authenticated") self.apply_options()
def btn_load_config_clicked(self): """ Opens a FileSelectionDialog and loads the selected config. """ fn = QtGui.QFileDialog.getOpenFileName(self, "Open Config-File...", os.getcwd(), "JSON-Files (*.json)") if fn == "": # cancel EventLogger.debug("Cancelled save Config.") return config_json = None try: with codecs.open(fn, 'r', 'UTF-8') as content_file: try: config_json = json.load(content_file) except ValueError as e: EventLogger.warning( "Load Config - Cant parse the configuration file: " + str(e)) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) return EventLogger.info("Loaded Config-File from: " + str(fn)) # devices config_blueprint = GuiConfigHandler.load_devices(config_json) if config_blueprint is None: return self.create_tree_items(config_blueprint) # general_section from brickv.data_logger.configuration_validator import ConfigurationReader self.update_setup_tab(config_json[ConfigurationReader.GENERAL_SECTION])
def validate(self): """ This function performs the validation of the various sections of the json configuration file """ EventLogger.info("Started configuration file validation") self.validate_general_section() self.validate_xively_section() self.validate_devices_section() EventLogger.info("Validation ends with [" + str(self._error_count) + "] errors") logging_time = self._log_space_counter.calculate_time() if self._log_space_counter.file_size != 0: EventLogger.info("Logging time until old data will be overwritten.") EventLogger.info("Days: " + str(logging_time[0]) + " Hours: " + str(logging_time[1]) + " Minutes: " + str(logging_time[2]) + " Seconds: " + str(logging_time[3])) EventLogger.info("Will write about " + str( int(self._log_space_counter.lines_per_second + 0.5)) + " lines per second into the log-file.") if self._error_count != 0: raise DataLoggerException(DataLoggerException.DL_FAILED_VALIDATION, "Validation process found some errors")
def btn_save_config_clicked(self): """ Opens a FileSelectionDialog and saves the current config. """ conf = GuiConfigHandler.create_config_file(self) fn = QtGui.QFileDialog.getSaveFileName(self, 'Save Config-File', os.getcwd(), filter='*.json') if fn == "": # cancel EventLogger.debug("Cancelled load Config.") return try: with open(fn, 'w') as outfile: json.dump(conf, outfile, sort_keys=True, indent=2) except Exception as e1: EventLogger.warning("Load Config - Exception: " + str(e1)) QMessageBox.warning(self, 'Error', 'Could not save the Config-File! Look at the Log-File for further information.', QMessageBox.Ok) return QMessageBox.information(self, 'Success', 'Config-File saved!', QMessageBox.Ok) EventLogger.info("Config-File saved to: " + str(fn))
def cb_connected(self, connect_reason): if self.secret != None: try: self.secret.encode('ascii') except: try: self.ipcon.disconnect() except: pass EventLogger.critical('Authentication secret cannot contain non-ASCII characters') return self.ipcon.set_auto_reconnect(False) # don't auto-reconnect on authentication error try: self.ipcon.authenticate(self.secret) except: try: self.ipcon.disconnect() except: pass if connect_reason == IPConnection.CONNECT_REASON_AUTO_RECONNECT: extra = ' after auto-reconnect' else: extra = '' EventLogger.critical('Could not authenticate' + extra) return self.ipcon.set_auto_reconnect(True) EventLogger.info("Successfully authenticated") self.apply_options()
def add_device_to_tree(self, device): # check if device is already added if len(device['uid']) > 0: for row in range(self.model_devices.rowCount()): existing_name = self.model_devices.item(row, 0).text() exisitng_uid = self.tree_devices.indexWidget(self.model_devices.item(row, 1).index()).text() if device['name'] == existing_name and device['uid'] == exisitng_uid: EventLogger.info('Ignoring duplicate device "{0}" with UID "{1}"' .format(device['name'], device['uid'])) return # add device name_item = QStandardItem(device['name']) uid_item = QStandardItem('') self.model_devices.appendRow([name_item, uid_item]) edit_uid = QLineEdit() edit_uid.setPlaceholderText('Enter UID') edit_uid.setValidator(QRegExpValidator(QRegExp('^[{0}]{{1,6}}$'.format(BASE58)))) # FIXME: use stricter logic edit_uid.setText(device['uid']) self.tree_devices.setIndexWidget(uid_item.index(), edit_uid) value_specs = device_specs[device['name']]['values'] parent_item = QStandardItem('Values') name_item.appendRow([parent_item, QStandardItem('')]) self.tree_devices.expand(parent_item.index()) # add values for value_spec in value_specs: value_name_item = QStandardItem(value_spec['name']) value_interval_item = QStandardItem('') parent_item.appendRow([value_name_item, value_interval_item]) spinbox_interval = QSpinBox() spinbox_interval.setRange(0, (1 << 31) - 1) spinbox_interval.setSingleStep(1) spinbox_interval.setValue(device['values'][value_spec['name']]['interval']) spinbox_interval.setSuffix(' seconds') self.tree_devices.setIndexWidget(value_interval_item.index(), spinbox_interval) if value_spec['subvalues'] != None: for subvalue_name in value_spec['subvalues']: subvalue_name_item = QStandardItem(subvalue_name) subvalue_check_item = QStandardItem('') value_name_item.appendRow([subvalue_name_item, subvalue_check_item]) check_subvalue = QCheckBox() check_subvalue.setChecked(device['values'][value_spec['name']]['subvalues'][subvalue_name]) self.tree_devices.setIndexWidget(subvalue_check_item.index(), check_subvalue) self.tree_devices.expand(name_item.index()) # add options option_specs = device_specs[device['name']]['options'] if option_specs != None: parent_item = QStandardItem('Options') name_item.appendRow([parent_item, QStandardItem('')]) for option_spec in option_specs: option_name_item = QStandardItem(option_spec['name']) option_widget_item = QStandardItem('') parent_item.appendRow([option_name_item, option_widget_item]) if option_spec['type'] == 'choice': widget_option_value = QComboBox() for option_value_spec in option_spec['values']: widget_option_value.addItem(option_value_spec[0].decode('utf-8'), option_value_spec[1]) widget_option_value.setCurrentIndex(widget_option_value.findText(device['options'][option_spec['name']]['value'].decode('utf-8'))) elif option_spec['type'] == 'int': widget_option_value = QSpinBox() widget_option_value.setRange(option_spec['minimum'], option_spec['maximum']) widget_option_value.setSuffix(option_spec['suffix']) widget_option_value.setValue(device['options'][option_spec['name']]['value']) elif option_spec['type'] == 'bool': widget_option_value = QCheckBox() widget_option_value.setChecked(device['options'][option_spec['name']]['value']) self.tree_devices.setIndexWidget(option_widget_item.index(), widget_option_value)
def add_device_to_tree(self, device): # check if device is already added if len(device['uid']) > 0: for row in range(self.model_devices.rowCount()): existing_name = self.model_devices.item(row, 0).text() exisitng_uid = self.tree_devices.indexWidget( self.model_devices.item(row, 1).index()).text() if device['name'] == existing_name and device[ 'uid'] == exisitng_uid: EventLogger.info( 'Ignoring duplicate device "{0}" with UID "{1}"'. format(device['name'], device['uid'])) return # add device name_item = QStandardItem(device['name']) uid_item = QStandardItem('') self.model_devices.appendRow([name_item, uid_item]) edit_uid = QLineEdit() edit_uid.setPlaceholderText('Enter UID') edit_uid.setValidator( QRegExpValidator(QRegExp( '^[{0}]{{1,6}}$'.format(BASE58)))) # FIXME: use stricter logic edit_uid.setText(device['uid']) self.tree_devices.setIndexWidget(uid_item.index(), edit_uid) value_specs = device_specs[device['name']]['values'] parent_item = QStandardItem('Values') name_item.appendRow([parent_item, QStandardItem('')]) self.tree_devices.expand(parent_item.index()) # add values for value_spec in value_specs: value_name_item = QStandardItem(value_spec['name']) value_interval_item = QStandardItem('') parent_item.appendRow([value_name_item, value_interval_item]) spinbox_interval = QSpinBox() spinbox_interval.setRange(0, (1 << 31) - 1) spinbox_interval.setSingleStep(1) spinbox_interval.setValue( device['values'][value_spec['name']]['interval']) spinbox_interval.setSuffix(' seconds') self.tree_devices.setIndexWidget(value_interval_item.index(), spinbox_interval) if value_spec['subvalues'] != None: for subvalue_name in value_spec['subvalues']: subvalue_name_item = QStandardItem(subvalue_name) subvalue_check_item = QStandardItem('') value_name_item.appendRow( [subvalue_name_item, subvalue_check_item]) check_subvalue = QCheckBox() check_subvalue.setChecked(device['values'][ value_spec['name']]['subvalues'][subvalue_name]) self.tree_devices.setIndexWidget( subvalue_check_item.index(), check_subvalue) self.tree_devices.expand(name_item.index()) # add options option_specs = device_specs[device['name']]['options'] if option_specs != None: parent_item = QStandardItem('Options') name_item.appendRow([parent_item, QStandardItem('')]) for option_spec in option_specs: option_name_item = QStandardItem(option_spec['name']) option_widget_item = QStandardItem('') parent_item.appendRow([option_name_item, option_widget_item]) if option_spec['type'] == 'choice': widget_option_value = QComboBox() for option_value_spec in option_spec['values']: widget_option_value.addItem( option_value_spec[0].decode('utf-8'), option_value_spec[1]) widget_option_value.setCurrentIndex( widget_option_value.findText(device['options'][ option_spec['name']]['value'].decode('utf-8'))) elif option_spec['type'] == 'int': widget_option_value = QSpinBox() widget_option_value.setRange(option_spec['minimum'], option_spec['maximum']) widget_option_value.setSuffix(option_spec['suffix']) widget_option_value.setValue( device['options'][option_spec['name']]['value']) elif option_spec['type'] == 'bool': widget_option_value = QCheckBox() widget_option_value.setChecked( device['options'][option_spec['name']]['value']) self.tree_devices.setIndexWidget(option_widget_item.index(), widget_option_value)