def test_creationDSsAndRRAs(self):
dss1 = []
rras1 = []
filename = '/tmp/test1.rrd'
dss1.append(DataSource(dsName='speed', dsType='COUNTER',
heartbeat=600))
rras1.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24))
rras1.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10))
rrd1 = RRD(filename, ds=dss1, rra=rras1, start=920804400)
self.assertEqual(repr(rrd1.ds), "[DS:speed:COUNTER:600:U:U]")
self.assertEqual(repr(rrd1.rra),
"[RRA:AVERAGE:0.5:1:24, RRA:AVERAGE:0.5:6:10]")
filename = '/tmp/test2.rrd'
rrd2 = RRD(filename, start=920804400)
self.assertEqual(rrd2.ds, [])
self.assertEqual(rrd2.rra, [])
dss3 = []
rras3 = []
filename = '/tmp/test3.rrd'
dss3.append(DataSource(dsName='speed', dsType='COUNTER',
heartbeat=300))
rras3.append(RRA(cf='AVERAGE', xff=0.5, steps=2, rows=24))
rras3.append(RRA(cf='AVERAGE', xff=0.5, steps=12, rows=10))
rrd3 = RRD(filename, ds=dss3, rra=rras3, start=920804400)
self.assertEqual(repr(rrd3.ds), "[DS:speed:COUNTER:300:U:U]")
self.assertEqual(repr(rrd3.rra),
"[RRA:AVERAGE:0.5:2:24, RRA:AVERAGE:0.5:12:10]")
def RrdCreate(rrdfile):
'''Creates a RRD database.'''
dataSources = []
roundRobinArchives = []
dataSources.append(
DataSource(dsName='temperature',
dsType='GAUGE',
heartbeat=600,
minval=-50,
maxval=100))
dataSources.append(
DataSource(dsName='humidity',
dsType='GAUGE',
heartbeat=600,
minval=0,
maxval=100))
dataSources.append(DataSource(dsName='mq9', dsType='GAUGE', heartbeat=600))
dataSources.append(
DataSource(dsName='dust_pc', dsType='GAUGE', heartbeat=600, minval=0))
dataSources.append(
DataSource(dsName='dust_raw', dsType='GAUGE', heartbeat=600))
# Keep all values for 10 days
roundRobinArchives.append(
RRA(cf='AVERAGE', xff=0.5, steps=1, rows=10 * 24 * 60))
# Keep 15-minute averages for one year days
roundRobinArchives.append(
RRA(cf='AVERAGE', xff=0.5, steps=15, rows=365 * 24 * 4))
# Keep 1-hour averages for 10 years
roundRobinArchives.append(
RRA(cf='AVERAGE', xff=0.5, steps=60, rows=10 * 365 * 24))
myRRD = RRD(rrdfile, step=60, ds=dataSources, rra=roundRobinArchives)
myRRD.create()
def __init__(self):
self.time = 1164783600 # small numbers of seconds since the epoch confuse rrdtool
self.prevstamptime = int(self.time)
ds = DataSource(ds_name='utilizationds', ds_type='GAUGE', heartbeat=1)
rra = RRA(cf='AVERAGE', xff=0.1, steps=1, rows=1200)
self.rrd = RRD("/tmp/utilization.rrd",
ds=[ds],
rra=[rra],
start=self.time)
self.rrd.create()
self.introducer = q = Introducer(self)
self.all_nodes = [
Node(randomid(), q, self) for i in range(self.NUM_NODES)
]
q.all_nodes = self.all_nodes
self.next = []
self.schedule_events()
self.verbose = False
self.added_files = 0
self.added_data = 0
self.deleted_files = 0
self.published_files = []
self.failed_files = 0
self.lost_data_bytes = 0 # bytes deleted to make room for new shares
def main(self, argv):
"""
Create an RRD file with values 0-9 entered at 1 second intervals from
1980-01-01 00:00:00 (the first date that rrdtool allows)
"""
from pyrrd.rrd import DataSource, RRA, RRD
start = int(datetime(1980, 1, 1, 0, 0).strftime('%s'))
dss = []
rras = []
filename = os.path.join(self.build_dir, 'test.rrd')
rows = 12
step = 10
dss.append(
DataSource(dsName='speed', dsType='GAUGE', heartbeat=2 * step))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=rows))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=12, rows=rows))
my_rrd = RRD(filename, ds=dss, rra=rras, start=start, step=step)
my_rrd.create()
for i, t in enumerate(
range(start + step, start + step + (rows * step), step)):
self.log.debug('DATA: %s %s (%s)' %
(t, i, datetime.fromtimestamp(t)))
my_rrd.bufferValue(t, i)
# Add further data 1 second later to demonstrate that the rrd
# lastupdatetime does not necessarily fall on a step boundary
t += 1
i += 1
self.log.debug('DATA: %s %s (%s)' % (t, i, datetime.fromtimestamp(t)))
my_rrd.bufferValue(t, i)
my_rrd.update()
def init_rdd(self):
# Initiates RRD-archive
# Creates the new one if absent or need to reset
filename = options.rrd_file
if not options.rrd_reset and access(filename, F_OK):
myRRD = RRD(filename)
else:
heartbeat = options.stats_period * 2
dataSources = [
DataSource(dsName='agents_u',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='t_unique',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='t_started',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='t_completed',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='t_failed',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='bytes',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='cpu',
dsType='DERIVE',
heartbeat=heartbeat,
minval=0),
DataSource(dsName='duration',
dsType='ABSOLUTE',
heartbeat=heartbeat),
DataSource(dsName='duration_avg',
dsType='GAUGE',
heartbeat=heartbeat),
]
roundRobinArchives = []
for (_steps, _rows) in options.rrd_rra:
roundRobinArchives.append(
RRA(cf='AVERAGE', xff=0.5, steps=_steps, rows=_rows))
roundRobinArchives.append(
RRA(cf='MAX', xff=0.5, steps=_steps, rows=_rows))
myRRD = RRD(filename,
ds=dataSources,
rra=roundRobinArchives,
step=options.stats_period)
myRRD.create(debug=True)
return myRRD
def setUp(self):
ds = [DataSource(dsName="speed", dsType="COUNTER", heartbeat=600)]
rra = [
RRA(cf="AVERAGE", xff=0.5, steps=1, rows=24),
RRA(cf="AVERAGE", xff=0.5, steps=6, rows=10)
]
self.rrdfile = tempfile.NamedTemporaryFile()
self.rrd = RRD(self.rrdfile.name, ds=ds, rra=rra, start=920804400)
self.rrd.create()
def main(args):
filename = 'test.rrd'
if not os.path.exists(filename):
dataSources = []
roundRobinArchives = []
dataSource = DataSource(dsName='speed',
dsType='COUNTER',
heartbeat=600)
print "dataSource.name:", dataSource.name
dataSources.append(dataSource)
roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24))
roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10))
myRRD = RRD(filename,
ds=dataSources,
rra=roundRobinArchives,
start=time.time())
myRRD.create()
#myRRD.update()
#write_2_file(myRRD)
else:
import random
myRRD = RRD(filename)
myRRD.bufferValue(time.time(), random.randrange(12393, 12423))
#=======================================================================
# myRRD.bufferValue('920805900', '12363')
# myRRD.bufferValue('920806200', '12373')
# myRRD.bufferValue('920806500', '12383')
# myRRD.update()
#
# myRRD.bufferValue('920806800', '12393')
# myRRD.bufferValue('920807100', '12399')
# myRRD.bufferValue('920807400', '12405')
# myRRD.bufferValue('920807700', '12411')
# myRRD.bufferValue('920808000', '12415')
# myRRD.bufferValue('920808300', '12420')
# myRRD.bufferValue('920808600', '12422')
# myRRD.bufferValue('920808900', '12423')
#=======================================================================
myRRD.update()
#write_2_file(myRRD)
print os.path.isfile(filename)
print len(open(filename).read())
def create_rrd(self, gpuid):
ret = RRD('/'.join([self.rrd_dir, '%s.rrd' % gpuid]),
ds=[
DataSource(dsName='utilization',
dsType='GAUGE',
heartbeat=10)
],
rra=[RRA(cf='MIN', xff=0.5, steps=1, rows=360)],
step=10,
start=int(time()))
ret.create()
return ret
def create_rrd(filename, args):
dses = []
rras = []
for ds in args['ds']:
dses.append(DataSource(dsName=ds, dsType=args['ds'][ds],
heartbeat=600))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=288))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=12, rows=744))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=24, rows=1116))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=48, rows=2191))
myRRD = RRD(filename, ds=dses, rra=rras, start=int(now) - 60)
myRRD.create()
def test_creation(self):
dss = []
rras = []
filename = '/tmp/test.rrd'
dss.append(DataSource(dsName='speed', dsType='COUNTER', heartbeat=600))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24))
rras.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10))
rrd = RRD(filename, ds=dss, rra=rras, start=920804400)
self.assertEqual(rrd.filename, filename)
self.assertEqual(repr(rrd.ds), "[DS:speed:COUNTER:600:U:U]")
self.assertEqual(repr(rrd.rra),
"[RRA:AVERAGE:0.5:1:24, RRA:AVERAGE:0.5:6:10]")
self.assertEqual(rrd.values, [])
self.assertEqual(rrd.step, 300)
self.assertEqual(rrd.lastupdate, None)
from pyrrd.rrd import DataSource, RRA, RRD
filename = '/tmp/test.rrd'
dataSources = []
roundRobinArchives = []
dataSource = DataSource(dsName='speed', dsType='COUNTER', heartbeat=600)
dataSources.append(dataSource)
roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24))
roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10))
myRRD = RRD(filename, ds=dataSources, rra=roundRobinArchives, start=920804400)
myRRD.create()
import os
print os.path.isfile(filename)
print len(open(filename).read())
myRRD.bufferValue('920805600', '12363')
myRRD.bufferValue('920805900', '12363')
myRRD.bufferValue('920806200', '12373')
myRRD.bufferValue('920806500', '12383')
myRRD.update()
myRRD.bufferValue('920806800', '12393')
myRRD.bufferValue('920807100', '12399')
myRRD.bufferValue('920807400', '12405')
myRRD.bufferValue('920807700', '12411')
myRRD.bufferValue('920808000', '12415')
myRRD.bufferValue('920808300', '12420')
myRRD.bufferValue('920808600', '12422')
myRRD.bufferValue('920808900', '12423')
myRRD.update()
def rrdtool_create(self, filename):
""" Create an rrdtool database if it doesn't exist """
# Create the directory if it doesn't exist.
directory = '/'.join(filename.split('/')[:-1])
if not os.path.isdir(directory):
os.makedirs(directory)
# Step interval for Primary Data Points (pdp)
pdp_step = self.frequency.seconds + (self.frequency.days * 86400)
# Heartbeat can be 'whatev', but twice the pdpd_step is good
heartbeat = 2 * pdp_step
# We only keep a single simple datasource.
sources = [
DataSource(
dsName='sum',
dsType='GAUGE',
heartbeat=heartbeat
)
]
# TODO -- this should be a user-definable number. It is equivalent to
# "how many data points do I want to see on any one graph at any given
# time." The higher it is, the cooler your graphs look. The higher it
# is, the more disk space is consumed. The higher it is, the more
# memory is consumed client-side.
target_resolution = 60
# This function calculates how many PDP steps should be involved in the
# calculation of a single Consolidated Data Point (CDP).
cdp_steps = lambda tspan : (tspan / pdp_step) / target_resolution
# Just a lookup of how many seconds per 'timespan'
timespans = {
'hour' : 3600,
'day' : 86400,
'week' : 604800,
'month' : 2629744,
'quarter' : 7889231,
'year' : 31556926,
}
self.log.info("Building rrd %s. %i cdp steps per hour." % (
filename, cdp_steps(timespans['hour'])))
# Here we build a series of round robin Archives of various resolutions
# and consolidation functions
archives = []
for consolidation_function in ['AVERAGE', 'MAX']:
archives += [
RRA(cf=consolidation_function, xff=0.5, rows=target_resolution,
steps=cdp_steps(seconds_per_timespan))
for name, seconds_per_timespan in timespans.iteritems()
]
# Actually build the round robin database from the parameters we built
rrd = RRD(
filename,
start=int(time.time()),
step=pdp_step,
ds=sources,
rra=archives,
)
rrd.create()
def main():
ns = parser.parse_args()
signal.signal(signal.SIGINT, sigint_handler)
dss = [
DataSource(dsName='vsz', dsType='GAUGE', heartbeat=2),
DataSource(dsName='rss', dsType='GAUGE', heartbeat=2)
]
rras = [
RRA(cf='AVERAGE', xff=0.5, steps=10, rows=3000),
#RRA(cf='AVERAGE', xff=0.5, steps=1, rows=100)
RRA(cf='LAST', xff=0.5, steps=1, rows=300000)
]
try:
os.remove(rrd_file)
os.remove(graph_file)
except OSError:
pass
rrd = RRD(rrd_file, ds=dss, rra=rras, step=1)
rrd.create()
start = time.time()
print("Starting at %d." % start)
while KEEP_GOING:
if ns.children:
vsz, rss = pid_and_subs_memory(ns.pid)
else:
vsz, rss = pid_memory(ns.pid)
#print("sample {} {}".format(size, rss))
if vsz == 0 and rss == 0:
break
rrd.bufferValue(time.time(), vsz, rss)
rrd.update()
time.sleep(1)
end = time.time()
print("Sampling finishes: %d." % end)
# #rrdtool fetch foo.rrd AVERAGE --end=now --start=now-50s
# command = [
# 'rrdtool',
# 'fetch',
# rrd_file,
# 'AVERAGE',
# '--end',
# str(int(end)),
# '--start',
# str(int(start))
# ]
# ps = subprocess.Popen(command)
# ps.wait()
#CDEF:mem_used_x=mem_used,1024,\* \
#LINE2:mem_used_x#D7CC00:mem_used
command = [
'rrdtool',
'graph',
'--title',
ns.graph_name,
graph_file,
'--start',
str(int(start)),
'--end',
str(int(end)),
# 'DEF:vsz={}:vsz:AVERAGE'.format(rrd_file),
'DEF:rss={}:rss:AVERAGE'.format(rrd_file),
# 'CDEF:vsz_k=vsz,1024,*',
'CDEF:rss_k=rss,1024,*',
# 'LINE:vsz_k#4287f5:Virtual',
'LINE:rss_k#42d7f5:Residential',
]
ps = subprocess.check_output(command)
print(ps)
sys.exit(0)
