def update(node, conf, archive, *values, **kwargs):
"""
Updates an existing RRD archive or creates a new one if needed.
"""
if not os.path.isfile(archive):
RRA.create(conf, archive)
nvalues = []
for idx, value in enumerate(values):
if value is None:
nvalues.append("U")
else:
nvalues.append(str(value))
rrdtool.update(archive, "{0}:{1}".format(kwargs.get("timestamp", "N"), ":".join(nvalues)))
# Record data in database store if set
data = {}
for i, x in enumerate(conf.sources):
data[x.name] = values[i]
now = time.time()
if "db_model" in conf.__dict__ and now - conf.last_update >= conf.interval:
try:
m = conf.db_model(**data)
m.node = node
m.timestamp = datetime.now()
m.save()
conf.last_update = now
except ValueError:
pass
# Record data in archive when available
if kwargs.get("graph") is not None:
data_archive.record_data(kwargs.get("graph"), datetime.now(), data)
def update(node, conf, archive, *values, **kwargs):
"""
Updates an existing RRD archive or creates a new one if needed.
"""
if not os.path.isfile(archive):
RRA.create(conf, archive)
nvalues = []
for idx, value in enumerate(values):
if value is None:
nvalues.append("U")
else:
nvalues.append(str(value))
rrdtool.update(
archive, "{0}:{1}".format(kwargs.get("timestamp", "N"),
":".join(nvalues)))
# Record data in database store if set
data = {}
for i, x in enumerate(conf.sources):
data[x.name] = values[i]
now = time.time()
if 'db_model' in conf.__dict__ and now - conf.last_update >= conf.interval:
try:
m = conf.db_model(**data)
m.node = node
m.timestamp = datetime.now()
m.save()
conf.last_update = now
except ValueError:
pass
# Record data in archive when available
if kwargs.get('graph') is not None:
data_archive.record_data(kwargs.get('graph'), datetime.now(), data)
def convert(conf, archive, action="refresh", opts="", graph=None):
"""
Converts
"""
try:
os.stat(archive)
except OSError:
return
# Dump the archive into XML form
process = subprocess.Popen(['/usr/bin/rrdtool', 'dump', archive],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Make any transformations needed
xml = ElementTree.parse(process.stdout)
wanted_source_names = [source.name for source in conf.sources]
start_offset = 5 # version, step, seconds comment, lastupdate, timestamp comment
changed = False
if action == "refresh":
# A simple marker to rescan sources as something has been modified
class RescanSources(Exception):
pass
while True:
data_sources = xml.findall('/ds')
data_source_names = [
source.findtext('./name').strip()
for source in data_sources
]
try:
for ds in data_source_names:
if ds not in wanted_source_names:
print "WARNING: Removal of sources currently not supported!"
# If we remove something we must continue the loop
# raise RescanSources
for idx, ds in enumerate(wanted_source_names):
if ds not in data_source_names:
print "INFO: Adding data source '%s' to RRD '%s.'" % (
ds, os.path.basename(archive))
# Update header
dse = ElementTree.Element("ds")
ElementTree.SubElement(
dse, "name").text = conf.sources[idx].name
ElementTree.SubElement(
dse, "type").text = conf.sources[idx].type
ElementTree.SubElement(
dse, "minimal_heartbeat").text = str(
conf.sources[idx].heartbeat)
ElementTree.SubElement(dse, "min").text = "NaN"
ElementTree.SubElement(dse, "max").text = "NaN"
ElementTree.SubElement(dse,
"last_ds").text = "UNKN"
ElementTree.SubElement(
dse, "value").text = "0.0000000000e+00"
ElementTree.SubElement(dse,
"unknown_sec").text = "0"
xml.getroot().insert(start_offset + idx, dse)
# Update all RRAs
for rra in xml.findall('/rra'):
# Update RRA header
cdp = rra.find('./cdp_prep')
dse = ElementTree.Element("ds")
ElementTree.SubElement(
dse, "primary_value").text = "NaN"
ElementTree.SubElement(
dse, "secondary_value").text = "NaN"
ElementTree.SubElement(dse,
"value").text = "NaN"
ElementTree.SubElement(
dse, "unknown_datapoints").text = "0"
cdp.insert(idx, dse)
# Update all RRA datapoints
for row in rra.findall('./database/row'):
v = ElementTree.Element("v")
v.text = "NaN"
row.insert(idx, v)
changed = True
break
except RescanSources:
pass
# Add RRAs when they have changed (only addition is supported)
for wanted_rra in conf.archives:
for rra in xml.findall('/rra'):
cf = rra.findtext('./cf').strip()
steps = int(rra.findtext('./pdp_per_row').strip())
rows = len(rra.findall('./database/row'))
xff = float(rra.findtext('./params/xff').strip())
match = all([
cf == wanted_rra.cf, steps == wanted_rra.steps,
rows == wanted_rra.rows, xff == wanted_rra.xff
])
if match:
break
else:
# Not found in existing RRAs, we need to add it
print "INFO: Adding new RRA '%s' to '%s'." % (
wanted_rra, os.path.basename(archive))
changed = True
rra = ElementTree.SubElement(xml.getroot(), "rra")
ElementTree.SubElement(rra, "cf").text = wanted_rra.cf
ElementTree.SubElement(rra, "pdp_per_row").text = str(
wanted_rra.steps)
params = ElementTree.SubElement(rra, "params")
ElementTree.SubElement(params,
"xff").text = str(wanted_rra.xff)
cdp_prep = ElementTree.SubElement(rra, "cdp_prep")
for ds in conf.sources:
dse = ElementTree.SubElement(cdp_prep, "ds")
ElementTree.SubElement(dse,
"primary_value").text = "NaN"
ElementTree.SubElement(dse,
"secondary_value").text = "NaN"
ElementTree.SubElement(dse, "value").text = "NaN"
ElementTree.SubElement(dse,
"unknown_datapoints").text = "0"
database = ElementTree.SubElement(rra, "database")
for row in xrange(wanted_rra.rows):
row = ElementTree.SubElement(database, "row")
for v in xrange(len(conf.sources)):
ElementTree.SubElement(row, "v").text = "NaN"
elif action == "archive":
# Archives data from RRDs
print "INFO: Archiving RRD '%s.'" % os.path.basename(archive)
for rra in xml.findall('/rra'):
cf = rra.findtext('./cf').strip()
if cf == 'AVERAGE':
timestamp = None
for item in rra.findall('./database')[0]:
if item.tag == "row":
# Data
values = [
x.text.strip() for x in item.findall('./v')
]
data = {}
for i, x in enumerate(conf.sources):
data[x.name] = values[i]
if graph is not None:
data_archive.record_data(
graph, timestamp, data)
else:
# Comment
timestamp = datetime.fromtimestamp(
int(item.text.split("/")[1].strip()))
elif action == "switch_sources":
# Switches two data sources
try:
ar_name, ds1, ds2 = opts.split(',')
except ValueError:
print "ERROR: Invalid RRD update options for action switch_sources!"
print "USAGE: <archive_class>,<ds1>,<ds2>"
return
if conf.__name__ != ar_name:
return
num_switched = 0
for ds in xml.findall('/ds/name'):
if ds.text.strip() == ds1:
ds.text = ds2
num_switched += 1
elif ds.text.strip() == ds2:
ds.text = ds1
num_switched += 1
if num_switched != 2:
print "ERROR: Unable to switch names for '{0}'!".format(
os.path.basename(archive))
return
print "INFO: Switching '{0}' and '{1}' in '{2}'.".format(
ds1, ds2, os.path.basename(archive))
changed = True
else:
print "ERROR: Invalid RRD convert action '%s'!" % action
return
if changed:
try:
os.rename(archive, "%s__bak" % archive)
process = subprocess.Popen(
['/usr/bin/rrdtool', 'restore', '-', archive],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Fix all empty last_ds values otherwise rrdtool will complain on restore
for last_ds in xml.findall('/ds/last_ds'):
if not last_ds.text:
last_ds.text = "UNKN"
process.communicate(ElementTree.tostring(xml.getroot()))
if process.returncode != 0:
raise Exception
try:
os.unlink("%s__bak" % archive)
except:
pass
except:
os.rename("%s__bak" % archive, archive)
raise
def convert(conf, archive, action = "refresh", opts = "", graph = None):
"""
Converts
"""
try:
os.stat(archive)
except OSError:
return
# Dump the archive into XML form
process = subprocess.Popen(
['/usr/bin/rrdtool', 'dump', archive],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
# Make any transformations needed
xml = ElementTree.parse(process.stdout)
wanted_source_names = [source.name for source in conf.sources]
start_offset = 5 # version, step, seconds comment, lastupdate, timestamp comment
changed = False
if action == "refresh":
# A simple marker to rescan sources as something has been modified
class RescanSources(Exception): pass
while True:
data_sources = xml.findall('/ds')
data_source_names = [source.findtext('./name').strip() for source in data_sources]
try:
for ds in data_source_names:
if ds not in wanted_source_names:
print "WARNING: Removal of sources currently not supported!"
# If we remove something we must continue the loop
# raise RescanSources
for idx, ds in enumerate(wanted_source_names):
if ds not in data_source_names:
print "INFO: Adding data source '%s' to RRD '%s.'" % (ds, os.path.basename(archive))
# Update header
dse = ElementTree.Element("ds")
ElementTree.SubElement(dse, "name").text = conf.sources[idx].name
ElementTree.SubElement(dse, "type").text = conf.sources[idx].type
ElementTree.SubElement(dse, "minimal_heartbeat").text = str(conf.sources[idx].heartbeat)
ElementTree.SubElement(dse, "min").text = "NaN"
ElementTree.SubElement(dse, "max").text = "NaN"
ElementTree.SubElement(dse, "last_ds").text = "UNKN"
ElementTree.SubElement(dse, "value").text = "0.0000000000e+00"
ElementTree.SubElement(dse, "unknown_sec").text = "0"
xml.getroot().insert(start_offset + idx, dse)
# Update all RRAs
for rra in xml.findall('/rra'):
# Update RRA header
cdp = rra.find('./cdp_prep')
dse = ElementTree.Element("ds")
ElementTree.SubElement(dse, "primary_value").text = "NaN"
ElementTree.SubElement(dse, "secondary_value").text = "NaN"
ElementTree.SubElement(dse, "value").text = "NaN"
ElementTree.SubElement(dse, "unknown_datapoints").text = "0"
cdp.insert(idx, dse)
# Update all RRA datapoints
for row in rra.findall('./database/row'):
v = ElementTree.Element("v")
v.text = "NaN"
row.insert(idx, v)
changed = True
break
except RescanSources:
pass
# Add RRAs when they have changed (only addition is supported)
for wanted_rra in conf.archives:
for rra in xml.findall('/rra'):
cf = rra.findtext('./cf').strip()
steps = int(rra.findtext('./pdp_per_row').strip())
rows = len(rra.findall('./database/row'))
xff = float(rra.findtext('./params/xff').strip())
match = all([
cf == wanted_rra.cf,
steps == wanted_rra.steps,
rows == wanted_rra.rows,
xff == wanted_rra.xff
])
if match:
break
else:
# Not found in existing RRAs, we need to add it
print "INFO: Adding new RRA '%s' to '%s'." % (wanted_rra, os.path.basename(archive))
changed = True
rra = ElementTree.SubElement(xml.getroot(), "rra")
ElementTree.SubElement(rra, "cf").text = wanted_rra.cf
ElementTree.SubElement(rra, "pdp_per_row").text = str(wanted_rra.steps)
params = ElementTree.SubElement(rra, "params")
ElementTree.SubElement(params, "xff").text = str(wanted_rra.xff)
cdp_prep = ElementTree.SubElement(rra, "cdp_prep")
for ds in conf.sources:
dse = ElementTree.SubElement(cdp_prep, "ds")
ElementTree.SubElement(dse, "primary_value").text = "NaN"
ElementTree.SubElement(dse, "secondary_value").text = "NaN"
ElementTree.SubElement(dse, "value").text = "NaN"
ElementTree.SubElement(dse, "unknown_datapoints").text = "0"
database = ElementTree.SubElement(rra, "database")
for row in xrange(wanted_rra.rows):
row = ElementTree.SubElement(database, "row")
for v in xrange(len(conf.sources)):
ElementTree.SubElement(row, "v").text = "NaN"
elif action == "archive":
# Archives data from RRDs
print "INFO: Archiving RRD '%s.'" % os.path.basename(archive)
for rra in xml.findall('/rra'):
cf = rra.findtext('./cf').strip()
if cf == 'AVERAGE':
timestamp = None
for item in rra.findall('./database')[0]:
if item.tag == "row":
# Data
values = [x.text.strip() for x in item.findall('./v')]
data = {}
for i, x in enumerate(conf.sources):
data[x.name] = values[i]
if graph is not None:
data_archive.record_data(graph, timestamp, data)
else:
# Comment
timestamp = datetime.fromtimestamp(int(item.text.split("/")[1].strip()))
elif action == "switch_sources":
# Switches two data sources
try:
ar_name, ds1, ds2 = opts.split(',')
except ValueError:
print "ERROR: Invalid RRD update options for action switch_sources!"
print "USAGE: <archive_class>,<ds1>,<ds2>"
return
if conf.__name__ != ar_name:
return
num_switched = 0
for ds in xml.findall('/ds/name'):
if ds.text.strip() == ds1:
ds.text = ds2
num_switched += 1
elif ds.text.strip() == ds2:
ds.text = ds1
num_switched += 1
if num_switched != 2:
print "ERROR: Unable to switch names for '{0}'!".format(os.path.basename(archive))
return
print "INFO: Switching '{0}' and '{1}' in '{2}'.".format(ds1, ds2, os.path.basename(archive))
changed = True
else:
print "ERROR: Invalid RRD convert action '%s'!" % action
return
if changed:
try:
os.rename(archive, "%s__bak" % archive)
process = subprocess.Popen(
['/usr/bin/rrdtool', 'restore', '-', archive],
stdin = subprocess.PIPE,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
# Fix all empty last_ds values otherwise rrdtool will complain on restore
for last_ds in xml.findall('/ds/last_ds'):
if not last_ds.text:
last_ds.text = "UNKN"
process.communicate(ElementTree.tostring(xml.getroot()))
if process.returncode != 0:
raise Exception
try:
os.unlink("%s__bak" % archive)
except:
pass
except:
os.rename("%s__bak" % archive, archive)
raise