def import_from_xml_folder(self, folder):
raise DeprecationWarning
# build file list and grouping if necessary
file_list = os.listdir(folder)
grouped_files = defaultdict(list)
errors = []
progress_bar = ProgressBar(len(file_list))
for file in file_list:
fullname = os.path.join(folder, file)
parts = file.replace(".xml", "").split("-")
series_name = ".".join(parts[0:-2])
if self.settings.influxdb['group_fields']:
grouped_files[series_name].append((parts[-2], fullname))
else:
grouped_files[".".join([series_name, parts[-2]])].append(('value', fullname))
if self.settings.interactive:
show = raw_input("Would you like to see the prospective series and columns? y/[n]: ") or "n"
if show in ("y", "Y"):
for series_name in sorted(grouped_files):
print(" - {2}{0}{3}: {1}".format(series_name, [name for name, _ in grouped_files[series_name]], Color.GREEN, Color.CLEAR))
print("Importing {0} XML files".format(len(file_list)))
for series_name in grouped_files:
data = []
keys_name = ['time']
values = defaultdict(list)
for field, file in grouped_files[series_name]:
progress_bar.update()
keys_name.append(field)
content = read_xml_file(file)
[values[key].append(value) for key, value in content.items()]
# join data with time as first column
data.extend([[k]+v for k, v in values.items()])
try:
pass
# self.upload_values(series_name, keys_name, data)
except Exception as e:
errors.append(str(e))
continue
try:
self.validate_record(series_name, keys_name)
except Exception as e:
errors.append("Validation error in {0}: {1}".format(series_name, e))
if errors:
print("The following errors were detected while importing:")
for error in errors:
print(" {0} {1}".format(Symbol.NOK_RED, error))
def import_from_xml_folder(self, folder):
raise DeprecationWarning
# build file list and grouping if necessary
file_list = os.listdir(folder)
grouped_files = defaultdict(list)
errors = []
progress_bar = ProgressBar(len(file_list))
for file in file_list:
fullname = os.path.join(folder, file)
parts = file.replace(".xml", "").split("-")
series_name = ".".join(parts[0:-2])
if self.settings.influxdb['group_fields']:
grouped_files[series_name].append((parts[-2], fullname))
else:
grouped_files[".".join([series_name, parts[-2]])].append(
('value', fullname))
if self.settings.interactive:
show = raw_input(
"Would you like to see the prospective series and columns? y/[n]: "
) or "n"
if show in ("y", "Y"):
for series_name in sorted(grouped_files):
print(" - {2}{0}{3}: {1}".format(
series_name,
[name for name, _ in grouped_files[series_name]],
Color.GREEN, Color.CLEAR))
print("Importing {0} XML files".format(len(file_list)))
for series_name in grouped_files:
data = []
keys_name = ['time']
values = defaultdict(list)
for field, file in grouped_files[series_name]:
progress_bar.update()
keys_name.append(field)
content = read_xml_file(file)
[values[key].append(value) for key, value in content.items()]
# join data with time as first column
data.extend([[k] + v for k, v in values.items()])
try:
pass
# self.upload_values(series_name, keys_name, data)
except Exception as e:
errors.append(str(e))
continue
try:
self.validate_record(series_name, keys_name)
except Exception as e:
errors.append("Validation error in {0}: {1}".format(
series_name, e))
if errors:
print("The following errors were detected while importing:")
for error in errors:
print(" {0} {1}".format(Symbol.NOK_RED, error))
def import_from_xml(self):
print("\nUploading data to InfluxDB:")
progress_bar = ProgressBar(self.settings.nb_rrd_files*3) # nb_files * (read + upload + validate)
errors = []
def _upload_and_validate(measurement, tags, fields, packed_values):
try:
self.write_series(measurement, tags, fields, packed_values)
except Exception as e:
errors.append((Symbol.NOK_RED, "Error writing {0} to InfluxDB: {1}".format(measurement, e)))
return
finally:
progress_bar.update(len(fields)-1) # 'time' column ignored
try:
self.validate_record(measurement, fields)
except Exception as e:
errors.append((Symbol.WARN_YELLOW, "Validation error in {0}: {1}".format(measurement, e)))
finally:
progress_bar.update(len(fields)-1) # 'time' column ignored
try:
assert self.client and self.valid
except:
raise Exception("Not connected to a InfluxDB server")
else:
print(" {0} Connection to database \"{1}\" OK".format(Symbol.OK_GREEN, self.settings.influxdb['database']))
if self.settings.influxdb['group_fields']:
"""
In "group_fields" mode, all fields of a same plugin (ex: system, user, nice, idle... of CPU usage)
will be represented as columns of the same time series in InfluxDB.
Schema will be:
+----------------------+-------+----------+----------+-----------+
| time_series_name | col_0 | col_1 | col_2 | col_3 ... |
+----------------------+-------+----------+----------+-----------+
| domain.host.plugin | time | metric_1 | metric_2 | metric_3 |
| acadis.org.tesla.cpu | time | system | user | nice |
| ... | | | | |
+----------------------+-------+----------+----------+-----------+
"""
for domain, host, plugin in self.settings.iter_plugins():
_plugin = self.settings.domains[domain].hosts[host].plugins[plugin]
measurement = plugin
tags = {
"domain": domain,
"host": host,
"plugin": plugin
}
if _plugin.is_multigraph:
tags["is_multigraph"] = True
print(host, plugin)
field_names = ['time']
values = defaultdict(list)
values_with_time = []
for field in _plugin.fields:
_field = _plugin.fields[field]
if _field.rrd_exported:
field_names.append(field)
try:
content = read_xml_file(_field.xml_filename)
except Exception as e:
errors.append((Symbol.WARN_YELLOW, "Could not read file for {0}: {1}".format(field, e)))
else:
[values[key].append(value) for key, value in content.items()]
# keep track of influxdb storage info to allow 'fetch'
_field.influxdb_measurement = measurement
_field.influxdb_field = field
_field.xml_imported = True
# update progress bar [###### ] 42 %
progress_bar.update()
# join data with time as first column
values_with_time.extend([[k]+v for k, v in values.items()])
_upload_and_validate(measurement, tags, field_names, values_with_time)
else: # non grouping
"""
In "non grouped" mode, all fields of a same plugin will have a dedicated time series and the values
will be written to a 'value' column
Schema will be:
+-----------------------------+-------+-------+
| time_series_name | col_0 | col_1 |
+-----------------------------+-------+-------+
| domain.host.plugin.metric_1 | time | value |
| domain.host.plugin.metric_2 | time | value |
| acadis.org.tesla.cpu.system | time | value |
| ... | | |
+-----------------------------+-------+-------+
"""
for domain, host, plugin, field in self.settings.iter_fields():
_field = self.settings.domains[domain].hosts[host].plugins[plugin].fields[field]
if not _field.rrd_exported:
continue
measurement = field
tags = {
"domain": domain,
"host": host,
"plugin": plugin
}
field_names = ['time', 'value']
values = defaultdict(list)
values_with_time = []
_field.influxdb_measurement = measurement
_field.influxdb_field = 'value'
content = read_xml_file(_field.xml_filename)
[values[key].append(value) for key, value in content.items()]
_field.xml_imported = True
progress_bar.update()
# join data with time as first column
values_with_time.extend([[k]+v for k, v in values.items()])
_upload_and_validate(measurement, tags, field_names, values_with_time)
for error in errors:
print(" {} {}".format(error[0], error[1]))
def import_from_xml(self):
print("\nUploading data to InfluxDB:")
progress_bar = ProgressBar(self.settings.nb_rrd_files *
3) # nb_files * (read + upload + validate)
errors = []
def _upload_and_validate(measurement, tags, fields, packed_values):
try:
self.write_series(measurement, tags, fields, packed_values)
except Exception as e:
errors.append((Symbol.NOK_RED,
"Error writing {0} to InfluxDB: {1}".format(
measurement, e)))
return
finally:
progress_bar.update(len(fields) - 1) # 'time' column ignored
try:
self.validate_record(measurement, fields)
except Exception as e:
errors.append(
(Symbol.WARN_YELLOW,
"Validation error in {0}: {1}".format(measurement, e)))
finally:
progress_bar.update(len(fields) - 1) # 'time' column ignored
try:
assert self.client and self.valid
except:
raise Exception("Not connected to a InfluxDB server")
else:
print(" {0} Connection to database \"{1}\" OK".format(
Symbol.OK_GREEN, self.settings.influxdb['database']))
if self.settings.influxdb['group_fields']:
"""
In "group_fields" mode, all fields of a same plugin (ex: system, user, nice, idle... of CPU usage)
will be represented as columns of the same time series in InfluxDB.
Schema will be:
+----------------------+-------+----------+----------+-----------+
| time_series_name | col_0 | col_1 | col_2 | col_3 ... |
+----------------------+-------+----------+----------+-----------+
| domain.host.plugin | time | metric_1 | metric_2 | metric_3 |
| acadis.org.tesla.cpu | time | system | user | nice |
| ... | | | | |
+----------------------+-------+----------+----------+-----------+
"""
for domain, host, plugin in self.settings.iter_plugins():
_plugin = self.settings.domains[domain].hosts[host].plugins[
plugin]
measurement = plugin
tags = {"domain": domain, "host": host, "plugin": plugin}
if _plugin.is_multigraph:
tags["is_multigraph"] = True
print(host, plugin)
field_names = ['time']
values = defaultdict(list)
values_with_time = []
for field in _plugin.fields:
_field = _plugin.fields[field]
if _field.rrd_exported:
field_names.append(field)
try:
content = read_xml_file(_field.xml_filename)
except Exception as e:
errors.append(
(Symbol.WARN_YELLOW,
"Could not read file for {0}: {1}".format(
field, e)))
else:
[
values[key].append(value)
for key, value in content.items()
]
# keep track of influxdb storage info to allow 'fetch'
_field.influxdb_measurement = measurement
_field.influxdb_field = field
_field.xml_imported = True
# update progress bar [###### ] 42 %
progress_bar.update()
# join data with time as first column
values_with_time.extend([[k] + v for k, v in values.items()])
_upload_and_validate(measurement, tags, field_names,
values_with_time)
else: # non grouping
"""
In "non grouped" mode, all fields of a same plugin will have a dedicated time series and the values
will be written to a 'value' column
Schema will be:
+-----------------------------+-------+-------+
| time_series_name | col_0 | col_1 |
+-----------------------------+-------+-------+
| domain.host.plugin.metric_1 | time | value |
| domain.host.plugin.metric_2 | time | value |
| acadis.org.tesla.cpu.system | time | value |
| ... | | |
+-----------------------------+-------+-------+
"""
for domain, host, plugin, field in self.settings.iter_fields():
_field = self.settings.domains[domain].hosts[host].plugins[
plugin].fields[field]
if not _field.rrd_exported:
continue
measurement = field
tags = {"domain": domain, "host": host, "plugin": plugin}
field_names = ['time', 'value']
values = defaultdict(list)
values_with_time = []
_field.influxdb_measurement = measurement
_field.influxdb_field = 'value'
content = read_xml_file(_field.xml_filename)
[values[key].append(value) for key, value in content.items()]
_field.xml_imported = True
progress_bar.update()
# join data with time as first column
values_with_time.extend([[k] + v for k, v in values.items()])
_upload_and_validate(measurement, tags, field_names,
values_with_time)
for error in errors:
print(" {} {}".format(error[0], error[1]))
