def import_from_xml_folder(self, folder): raise DeprecationWarning # build file list and grouping if necessary file_list = os.listdir(folder) grouped_files = defaultdict(list) errors = [] progress_bar = ProgressBar(len(file_list)) for file in file_list: fullname = os.path.join(folder, file) parts = file.replace(".xml", "").split("-") series_name = ".".join(parts[0:-2]) if self.settings.influxdb['group_fields']: grouped_files[series_name].append((parts[-2], fullname)) else: grouped_files[".".join([series_name, parts[-2]])].append(('value', fullname)) if self.settings.interactive: show = raw_input("Would you like to see the prospective series and columns? y/[n]: ") or "n" if show in ("y", "Y"): for series_name in sorted(grouped_files): print(" - {2}{0}{3}: {1}".format(series_name, [name for name, _ in grouped_files[series_name]], Color.GREEN, Color.CLEAR)) print("Importing {0} XML files".format(len(file_list))) for series_name in grouped_files: data = [] keys_name = ['time'] values = defaultdict(list) for field, file in grouped_files[series_name]: progress_bar.update() keys_name.append(field) content = read_xml_file(file) [values[key].append(value) for key, value in content.items()] # join data with time as first column data.extend([[k]+v for k, v in values.items()]) try: pass # self.upload_values(series_name, keys_name, data) except Exception as e: errors.append(str(e)) continue try: self.validate_record(series_name, keys_name) except Exception as e: errors.append("Validation error in {0}: {1}".format(series_name, e)) if errors: print("The following errors were detected while importing:") for error in errors: print(" {0} {1}".format(Symbol.NOK_RED, error))
def import_from_xml_folder(self, folder): raise DeprecationWarning # build file list and grouping if necessary file_list = os.listdir(folder) grouped_files = defaultdict(list) errors = [] progress_bar = ProgressBar(len(file_list)) for file in file_list: fullname = os.path.join(folder, file) parts = file.replace(".xml", "").split("-") series_name = ".".join(parts[0:-2]) if self.settings.influxdb['group_fields']: grouped_files[series_name].append((parts[-2], fullname)) else: grouped_files[".".join([series_name, parts[-2]])].append( ('value', fullname)) if self.settings.interactive: show = raw_input( "Would you like to see the prospective series and columns? y/[n]: " ) or "n" if show in ("y", "Y"): for series_name in sorted(grouped_files): print(" - {2}{0}{3}: {1}".format( series_name, [name for name, _ in grouped_files[series_name]], Color.GREEN, Color.CLEAR)) print("Importing {0} XML files".format(len(file_list))) for series_name in grouped_files: data = [] keys_name = ['time'] values = defaultdict(list) for field, file in grouped_files[series_name]: progress_bar.update() keys_name.append(field) content = read_xml_file(file) [values[key].append(value) for key, value in content.items()] # join data with time as first column data.extend([[k] + v for k, v in values.items()]) try: pass # self.upload_values(series_name, keys_name, data) except Exception as e: errors.append(str(e)) continue try: self.validate_record(series_name, keys_name) except Exception as e: errors.append("Validation error in {0}: {1}".format( series_name, e)) if errors: print("The following errors were detected while importing:") for error in errors: print(" {0} {1}".format(Symbol.NOK_RED, error))
def import_from_xml(self): print("\nUploading data to InfluxDB:") progress_bar = ProgressBar(self.settings.nb_rrd_files*3) # nb_files * (read + upload + validate) errors = [] def _upload_and_validate(measurement, tags, fields, packed_values): try: self.write_series(measurement, tags, fields, packed_values) except Exception as e: errors.append((Symbol.NOK_RED, "Error writing {0} to InfluxDB: {1}".format(measurement, e))) return finally: progress_bar.update(len(fields)-1) # 'time' column ignored try: self.validate_record(measurement, fields) except Exception as e: errors.append((Symbol.WARN_YELLOW, "Validation error in {0}: {1}".format(measurement, e))) finally: progress_bar.update(len(fields)-1) # 'time' column ignored try: assert self.client and self.valid except: raise Exception("Not connected to a InfluxDB server") else: print(" {0} Connection to database \"{1}\" OK".format(Symbol.OK_GREEN, self.settings.influxdb['database'])) if self.settings.influxdb['group_fields']: """ In "group_fields" mode, all fields of a same plugin (ex: system, user, nice, idle... of CPU usage) will be represented as columns of the same time series in InfluxDB. Schema will be: +----------------------+-------+----------+----------+-----------+ | time_series_name | col_0 | col_1 | col_2 | col_3 ... | +----------------------+-------+----------+----------+-----------+ | domain.host.plugin | time | metric_1 | metric_2 | metric_3 | | acadis.org.tesla.cpu | time | system | user | nice | | ... | | | | | +----------------------+-------+----------+----------+-----------+ """ for domain, host, plugin in self.settings.iter_plugins(): _plugin = self.settings.domains[domain].hosts[host].plugins[plugin] measurement = plugin tags = { "domain": domain, "host": host, "plugin": plugin } if _plugin.is_multigraph: tags["is_multigraph"] = True print(host, plugin) field_names = ['time'] values = defaultdict(list) values_with_time = [] for field in _plugin.fields: _field = _plugin.fields[field] if _field.rrd_exported: field_names.append(field) try: content = read_xml_file(_field.xml_filename) except Exception as e: errors.append((Symbol.WARN_YELLOW, "Could not read file for {0}: {1}".format(field, e))) else: [values[key].append(value) for key, value in content.items()] # keep track of influxdb storage info to allow 'fetch' _field.influxdb_measurement = measurement _field.influxdb_field = field _field.xml_imported = True # update progress bar [###### ] 42 % progress_bar.update() # join data with time as first column values_with_time.extend([[k]+v for k, v in values.items()]) _upload_and_validate(measurement, tags, field_names, values_with_time) else: # non grouping """ In "non grouped" mode, all fields of a same plugin will have a dedicated time series and the values will be written to a 'value' column Schema will be: +-----------------------------+-------+-------+ | time_series_name | col_0 | col_1 | +-----------------------------+-------+-------+ | domain.host.plugin.metric_1 | time | value | | domain.host.plugin.metric_2 | time | value | | acadis.org.tesla.cpu.system | time | value | | ... | | | +-----------------------------+-------+-------+ """ for domain, host, plugin, field in self.settings.iter_fields(): _field = self.settings.domains[domain].hosts[host].plugins[plugin].fields[field] if not _field.rrd_exported: continue measurement = field tags = { "domain": domain, "host": host, "plugin": plugin } field_names = ['time', 'value'] values = defaultdict(list) values_with_time = [] _field.influxdb_measurement = measurement _field.influxdb_field = 'value' content = read_xml_file(_field.xml_filename) [values[key].append(value) for key, value in content.items()] _field.xml_imported = True progress_bar.update() # join data with time as first column values_with_time.extend([[k]+v for k, v in values.items()]) _upload_and_validate(measurement, tags, field_names, values_with_time) for error in errors: print(" {} {}".format(error[0], error[1]))
def import_from_xml(self): print("\nUploading data to InfluxDB:") progress_bar = ProgressBar(self.settings.nb_rrd_files * 3) # nb_files * (read + upload + validate) errors = [] def _upload_and_validate(measurement, tags, fields, packed_values): try: self.write_series(measurement, tags, fields, packed_values) except Exception as e: errors.append((Symbol.NOK_RED, "Error writing {0} to InfluxDB: {1}".format( measurement, e))) return finally: progress_bar.update(len(fields) - 1) # 'time' column ignored try: self.validate_record(measurement, fields) except Exception as e: errors.append( (Symbol.WARN_YELLOW, "Validation error in {0}: {1}".format(measurement, e))) finally: progress_bar.update(len(fields) - 1) # 'time' column ignored try: assert self.client and self.valid except: raise Exception("Not connected to a InfluxDB server") else: print(" {0} Connection to database \"{1}\" OK".format( Symbol.OK_GREEN, self.settings.influxdb['database'])) if self.settings.influxdb['group_fields']: """ In "group_fields" mode, all fields of a same plugin (ex: system, user, nice, idle... of CPU usage) will be represented as columns of the same time series in InfluxDB. Schema will be: +----------------------+-------+----------+----------+-----------+ | time_series_name | col_0 | col_1 | col_2 | col_3 ... | +----------------------+-------+----------+----------+-----------+ | domain.host.plugin | time | metric_1 | metric_2 | metric_3 | | acadis.org.tesla.cpu | time | system | user | nice | | ... | | | | | +----------------------+-------+----------+----------+-----------+ """ for domain, host, plugin in self.settings.iter_plugins(): _plugin = self.settings.domains[domain].hosts[host].plugins[ plugin] measurement = plugin tags = {"domain": domain, "host": host, "plugin": plugin} if _plugin.is_multigraph: tags["is_multigraph"] = True print(host, plugin) field_names = ['time'] values = defaultdict(list) values_with_time = [] for field in _plugin.fields: _field = _plugin.fields[field] if _field.rrd_exported: field_names.append(field) try: content = read_xml_file(_field.xml_filename) except Exception as e: errors.append( (Symbol.WARN_YELLOW, "Could not read file for {0}: {1}".format( field, e))) else: [ values[key].append(value) for key, value in content.items() ] # keep track of influxdb storage info to allow 'fetch' _field.influxdb_measurement = measurement _field.influxdb_field = field _field.xml_imported = True # update progress bar [###### ] 42 % progress_bar.update() # join data with time as first column values_with_time.extend([[k] + v for k, v in values.items()]) _upload_and_validate(measurement, tags, field_names, values_with_time) else: # non grouping """ In "non grouped" mode, all fields of a same plugin will have a dedicated time series and the values will be written to a 'value' column Schema will be: +-----------------------------+-------+-------+ | time_series_name | col_0 | col_1 | +-----------------------------+-------+-------+ | domain.host.plugin.metric_1 | time | value | | domain.host.plugin.metric_2 | time | value | | acadis.org.tesla.cpu.system | time | value | | ... | | | +-----------------------------+-------+-------+ """ for domain, host, plugin, field in self.settings.iter_fields(): _field = self.settings.domains[domain].hosts[host].plugins[ plugin].fields[field] if not _field.rrd_exported: continue measurement = field tags = {"domain": domain, "host": host, "plugin": plugin} field_names = ['time', 'value'] values = defaultdict(list) values_with_time = [] _field.influxdb_measurement = measurement _field.influxdb_field = 'value' content = read_xml_file(_field.xml_filename) [values[key].append(value) for key, value in content.items()] _field.xml_imported = True progress_bar.update() # join data with time as first column values_with_time.extend([[k] + v for k, v in values.items()]) _upload_and_validate(measurement, tags, field_names, values_with_time) for error in errors: print(" {} {}".format(error[0], error[1]))