def test_creationDSsAndRRAs(self): dss1 = [] rras1 = [] filename = '/tmp/test1.rrd' dss1.append(DataSource(dsName='speed', dsType='COUNTER', heartbeat=600)) rras1.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24)) rras1.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10)) rrd1 = RRD(filename, ds=dss1, rra=rras1, start=920804400) self.assertEqual(repr(rrd1.ds), "[DS:speed:COUNTER:600:U:U]") self.assertEqual(repr(rrd1.rra), "[RRA:AVERAGE:0.5:1:24, RRA:AVERAGE:0.5:6:10]") filename = '/tmp/test2.rrd' rrd2 = RRD(filename, start=920804400) self.assertEqual(rrd2.ds, []) self.assertEqual(rrd2.rra, []) dss3 = [] rras3 = [] filename = '/tmp/test3.rrd' dss3.append(DataSource(dsName='speed', dsType='COUNTER', heartbeat=300)) rras3.append(RRA(cf='AVERAGE', xff=0.5, steps=2, rows=24)) rras3.append(RRA(cf='AVERAGE', xff=0.5, steps=12, rows=10)) rrd3 = RRD(filename, ds=dss3, rra=rras3, start=920804400) self.assertEqual(repr(rrd3.ds), "[DS:speed:COUNTER:300:U:U]") self.assertEqual(repr(rrd3.rra), "[RRA:AVERAGE:0.5:2:24, RRA:AVERAGE:0.5:12:10]")
def RrdCreate(rrdfile): '''Creates a RRD database.''' dataSources = [] roundRobinArchives = [] dataSources.append( DataSource(dsName='temperature', dsType='GAUGE', heartbeat=600, minval=-50, maxval=100)) dataSources.append( DataSource(dsName='humidity', dsType='GAUGE', heartbeat=600, minval=0, maxval=100)) dataSources.append(DataSource(dsName='mq9', dsType='GAUGE', heartbeat=600)) dataSources.append( DataSource(dsName='dust_pc', dsType='GAUGE', heartbeat=600, minval=0)) dataSources.append( DataSource(dsName='dust_raw', dsType='GAUGE', heartbeat=600)) # Keep all values for 10 days roundRobinArchives.append( RRA(cf='AVERAGE', xff=0.5, steps=1, rows=10 * 24 * 60)) # Keep 15-minute averages for one year days roundRobinArchives.append( RRA(cf='AVERAGE', xff=0.5, steps=15, rows=365 * 24 * 4)) # Keep 1-hour averages for 10 years roundRobinArchives.append( RRA(cf='AVERAGE', xff=0.5, steps=60, rows=10 * 365 * 24)) myRRD = RRD(rrdfile, step=60, ds=dataSources, rra=roundRobinArchives) myRRD.create()
def __init__(self): self.time = 1164783600 # small numbers of seconds since the epoch confuse rrdtool self.prevstamptime = int(self.time) ds = DataSource(ds_name='utilizationds', ds_type='GAUGE', heartbeat=1) rra = RRA(cf='AVERAGE', xff=0.1, steps=1, rows=1200) self.rrd = RRD("/tmp/utilization.rrd", ds=[ds], rra=[rra], start=self.time) self.rrd.create() self.introducer = q = Introducer(self) self.all_nodes = [ Node(randomid(), q, self) for i in range(self.NUM_NODES) ] q.all_nodes = self.all_nodes self.next = [] self.schedule_events() self.verbose = False self.added_files = 0 self.added_data = 0 self.deleted_files = 0 self.published_files = [] self.failed_files = 0 self.lost_data_bytes = 0 # bytes deleted to make room for new shares
def main(self, argv): """ Create an RRD file with values 0-9 entered at 1 second intervals from 1980-01-01 00:00:00 (the first date that rrdtool allows) """ from pyrrd.rrd import DataSource, RRA, RRD start = int(datetime(1980, 1, 1, 0, 0).strftime('%s')) dss = [] rras = [] filename = os.path.join(self.build_dir, 'test.rrd') rows = 12 step = 10 dss.append( DataSource(dsName='speed', dsType='GAUGE', heartbeat=2 * step)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=rows)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=12, rows=rows)) my_rrd = RRD(filename, ds=dss, rra=rras, start=start, step=step) my_rrd.create() for i, t in enumerate( range(start + step, start + step + (rows * step), step)): self.log.debug('DATA: %s %s (%s)' % (t, i, datetime.fromtimestamp(t))) my_rrd.bufferValue(t, i) # Add further data 1 second later to demonstrate that the rrd # lastupdatetime does not necessarily fall on a step boundary t += 1 i += 1 self.log.debug('DATA: %s %s (%s)' % (t, i, datetime.fromtimestamp(t))) my_rrd.bufferValue(t, i) my_rrd.update()
def init_rdd(self): # Initiates RRD-archive # Creates the new one if absent or need to reset filename = options.rrd_file if not options.rrd_reset and access(filename, F_OK): myRRD = RRD(filename) else: heartbeat = options.stats_period * 2 dataSources = [ DataSource(dsName='agents_u', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='t_unique', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='t_started', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='t_completed', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='t_failed', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='bytes', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='cpu', dsType='DERIVE', heartbeat=heartbeat, minval=0), DataSource(dsName='duration', dsType='ABSOLUTE', heartbeat=heartbeat), DataSource(dsName='duration_avg', dsType='GAUGE', heartbeat=heartbeat), ] roundRobinArchives = [] for (_steps, _rows) in options.rrd_rra: roundRobinArchives.append( RRA(cf='AVERAGE', xff=0.5, steps=_steps, rows=_rows)) roundRobinArchives.append( RRA(cf='MAX', xff=0.5, steps=_steps, rows=_rows)) myRRD = RRD(filename, ds=dataSources, rra=roundRobinArchives, step=options.stats_period) myRRD.create(debug=True) return myRRD
def setUp(self): ds = [DataSource(dsName="speed", dsType="COUNTER", heartbeat=600)] rra = [ RRA(cf="AVERAGE", xff=0.5, steps=1, rows=24), RRA(cf="AVERAGE", xff=0.5, steps=6, rows=10) ] self.rrdfile = tempfile.NamedTemporaryFile() self.rrd = RRD(self.rrdfile.name, ds=ds, rra=rra, start=920804400) self.rrd.create()
def main(args): filename = 'test.rrd' if not os.path.exists(filename): dataSources = [] roundRobinArchives = [] dataSource = DataSource(dsName='speed', dsType='COUNTER', heartbeat=600) print "dataSource.name:", dataSource.name dataSources.append(dataSource) roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24)) roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10)) myRRD = RRD(filename, ds=dataSources, rra=roundRobinArchives, start=time.time()) myRRD.create() #myRRD.update() #write_2_file(myRRD) else: import random myRRD = RRD(filename) myRRD.bufferValue(time.time(), random.randrange(12393, 12423)) #======================================================================= # myRRD.bufferValue('920805900', '12363') # myRRD.bufferValue('920806200', '12373') # myRRD.bufferValue('920806500', '12383') # myRRD.update() # # myRRD.bufferValue('920806800', '12393') # myRRD.bufferValue('920807100', '12399') # myRRD.bufferValue('920807400', '12405') # myRRD.bufferValue('920807700', '12411') # myRRD.bufferValue('920808000', '12415') # myRRD.bufferValue('920808300', '12420') # myRRD.bufferValue('920808600', '12422') # myRRD.bufferValue('920808900', '12423') #======================================================================= myRRD.update() #write_2_file(myRRD) print os.path.isfile(filename) print len(open(filename).read())
def create_rrd(self, gpuid): ret = RRD('/'.join([self.rrd_dir, '%s.rrd' % gpuid]), ds=[ DataSource(dsName='utilization', dsType='GAUGE', heartbeat=10) ], rra=[RRA(cf='MIN', xff=0.5, steps=1, rows=360)], step=10, start=int(time())) ret.create() return ret
def create_rrd(filename, args): dses = [] rras = [] for ds in args['ds']: dses.append(DataSource(dsName=ds, dsType=args['ds'][ds], heartbeat=600)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=288)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=12, rows=744)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=24, rows=1116)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=48, rows=2191)) myRRD = RRD(filename, ds=dses, rra=rras, start=int(now) - 60) myRRD.create()
def test_creation(self): dss = [] rras = [] filename = '/tmp/test.rrd' dss.append(DataSource(dsName='speed', dsType='COUNTER', heartbeat=600)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24)) rras.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10)) rrd = RRD(filename, ds=dss, rra=rras, start=920804400) self.assertEqual(rrd.filename, filename) self.assertEqual(repr(rrd.ds), "[DS:speed:COUNTER:600:U:U]") self.assertEqual(repr(rrd.rra), "[RRA:AVERAGE:0.5:1:24, RRA:AVERAGE:0.5:6:10]") self.assertEqual(rrd.values, []) self.assertEqual(rrd.step, 300) self.assertEqual(rrd.lastupdate, None)
from pyrrd.rrd import DataSource, RRA, RRD filename = '/tmp/test.rrd' dataSources = [] roundRobinArchives = [] dataSource = DataSource(dsName='speed', dsType='COUNTER', heartbeat=600) dataSources.append(dataSource) roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=1, rows=24)) roundRobinArchives.append(RRA(cf='AVERAGE', xff=0.5, steps=6, rows=10)) myRRD = RRD(filename, ds=dataSources, rra=roundRobinArchives, start=920804400) myRRD.create() import os print os.path.isfile(filename) print len(open(filename).read()) myRRD.bufferValue('920805600', '12363') myRRD.bufferValue('920805900', '12363') myRRD.bufferValue('920806200', '12373') myRRD.bufferValue('920806500', '12383') myRRD.update() myRRD.bufferValue('920806800', '12393') myRRD.bufferValue('920807100', '12399') myRRD.bufferValue('920807400', '12405') myRRD.bufferValue('920807700', '12411') myRRD.bufferValue('920808000', '12415') myRRD.bufferValue('920808300', '12420') myRRD.bufferValue('920808600', '12422') myRRD.bufferValue('920808900', '12423') myRRD.update()
def rrdtool_create(self, filename): """ Create an rrdtool database if it doesn't exist """ # Create the directory if it doesn't exist. directory = '/'.join(filename.split('/')[:-1]) if not os.path.isdir(directory): os.makedirs(directory) # Step interval for Primary Data Points (pdp) pdp_step = self.frequency.seconds + (self.frequency.days * 86400) # Heartbeat can be 'whatev', but twice the pdpd_step is good heartbeat = 2 * pdp_step # We only keep a single simple datasource. sources = [ DataSource( dsName='sum', dsType='GAUGE', heartbeat=heartbeat ) ] # TODO -- this should be a user-definable number. It is equivalent to # "how many data points do I want to see on any one graph at any given # time." The higher it is, the cooler your graphs look. The higher it # is, the more disk space is consumed. The higher it is, the more # memory is consumed client-side. target_resolution = 60 # This function calculates how many PDP steps should be involved in the # calculation of a single Consolidated Data Point (CDP). cdp_steps = lambda tspan : (tspan / pdp_step) / target_resolution # Just a lookup of how many seconds per 'timespan' timespans = { 'hour' : 3600, 'day' : 86400, 'week' : 604800, 'month' : 2629744, 'quarter' : 7889231, 'year' : 31556926, } self.log.info("Building rrd %s. %i cdp steps per hour." % ( filename, cdp_steps(timespans['hour']))) # Here we build a series of round robin Archives of various resolutions # and consolidation functions archives = [] for consolidation_function in ['AVERAGE', 'MAX']: archives += [ RRA(cf=consolidation_function, xff=0.5, rows=target_resolution, steps=cdp_steps(seconds_per_timespan)) for name, seconds_per_timespan in timespans.iteritems() ] # Actually build the round robin database from the parameters we built rrd = RRD( filename, start=int(time.time()), step=pdp_step, ds=sources, rra=archives, ) rrd.create()
def main(): ns = parser.parse_args() signal.signal(signal.SIGINT, sigint_handler) dss = [ DataSource(dsName='vsz', dsType='GAUGE', heartbeat=2), DataSource(dsName='rss', dsType='GAUGE', heartbeat=2) ] rras = [ RRA(cf='AVERAGE', xff=0.5, steps=10, rows=3000), #RRA(cf='AVERAGE', xff=0.5, steps=1, rows=100) RRA(cf='LAST', xff=0.5, steps=1, rows=300000) ] try: os.remove(rrd_file) os.remove(graph_file) except OSError: pass rrd = RRD(rrd_file, ds=dss, rra=rras, step=1) rrd.create() start = time.time() print("Starting at %d." % start) while KEEP_GOING: if ns.children: vsz, rss = pid_and_subs_memory(ns.pid) else: vsz, rss = pid_memory(ns.pid) #print("sample {} {}".format(size, rss)) if vsz == 0 and rss == 0: break rrd.bufferValue(time.time(), vsz, rss) rrd.update() time.sleep(1) end = time.time() print("Sampling finishes: %d." % end) # #rrdtool fetch foo.rrd AVERAGE --end=now --start=now-50s # command = [ # 'rrdtool', # 'fetch', # rrd_file, # 'AVERAGE', # '--end', # str(int(end)), # '--start', # str(int(start)) # ] # ps = subprocess.Popen(command) # ps.wait() #CDEF:mem_used_x=mem_used,1024,\* \ #LINE2:mem_used_x#D7CC00:mem_used command = [ 'rrdtool', 'graph', '--title', ns.graph_name, graph_file, '--start', str(int(start)), '--end', str(int(end)), # 'DEF:vsz={}:vsz:AVERAGE'.format(rrd_file), 'DEF:rss={}:rss:AVERAGE'.format(rrd_file), # 'CDEF:vsz_k=vsz,1024,*', 'CDEF:rss_k=rss,1024,*', # 'LINE:vsz_k#4287f5:Virtual', 'LINE:rss_k#42d7f5:Residential', ] ps = subprocess.check_output(command) print(ps) sys.exit(0)