/
bwmon.py
722 lines (621 loc) · 26.9 KB
/
bwmon.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
#!/usr/bin/python3
#
# Average bandwidth monitoring script. Run periodically via NM db.sync to
# enforce a soft limit on daily bandwidth usage for each slice. If a
# slice is found to have transmitted 80% of its daily byte limit usage,
# its instantaneous rate will be capped at the bytes remaning in the limit
# over the time remaining in the recording period.
#
# Two separate limits are enforced, one for destinations exempt from
# the node bandwidth cap (i.e. Internet2), and the other for all other destinations.
#
# Mark Huang <mlhuang@cs.princeton.edu>
# Andy Bavier <acb@cs.princeton.edu>
# Faiyaz Ahmed <faiyaza@cs.princeton.edu>
# Copyright (C) 2004-2008 The Trustees of Princeton University
#
import os
import sys
import time
import pickle
import socket
import copy
import threading
import plnode.bwlimit as bwlimit
import logger
import tools
import database
from config import Config
priority = 20
# Defaults
# Set DEBUG to True if you don't want to send emails
DEBUG = False
# Set ENABLE to False to setup buckets, but not limit.
ENABLE = True
DB_FILE = "/var/lib/nodemanager/bwmon.pickle"
# Constants
seconds_per_day = 24 * 60 * 60
bits_per_byte = 8
dev_default = tools.get_default_if()
# Burst to line rate (or node cap). Set by NM. in KBit/s
default_MaxRate = int(bwlimit.get_bwcap(dev_default) / 1000)
default_Maxi2Rate = int(bwlimit.bwmax / 1000)
# 5.4 Gbyte per day. 5.4 * 1024 k * 1024M * 1024G
# 5.4 Gbyte per day max allowed transfered per recording period
# 5.4 Gbytes per day is aprox 512k/s for 24hrs (approx because original math was wrong
# but its better to keep a higher byte total and keep people happy than correct
# the problem and piss people off.
# default_MaxKByte = 5662310
# -- 6/1/09
# llp wants to double these, so we use the following
# 1mbit * 24hrs * 60mins * 60secs = bits/day
# 1000000 * 24 * 60 * 60 / (1024 * 8)
default_MaxKByte = 10546875
# 16.4 Gbyte per day max allowed transfered per recording period to I2
# default_Maxi2KByte = 17196646
# -- 6/1/09
# 3Mb/s for 24hrs a day (30.17 gigs)
default_Maxi2KByte = 31640625
# Default share quanta
default_Share = 1
# Average over 1 day
period = 1 * seconds_per_day
# Message template
template = \
"""
The slice %(slice)s has transmitted more than %(bytes)s from
%(hostname)s to %(class)s destinations
since %(since)s.
Its maximum %(class)s burst rate will be capped at %(new_maxrate)s/s
until %(until)s.
Please reduce the average %(class)s transmission rate
of the slice to %(limit)s per %(period)s.
""".lstrip()
footer = \
"""
%(date)s %(hostname)s bwcap %(slice)s
""".lstrip()
def format_bytes(bytes, si = True):
"""
Formats bytes into a string
"""
if si:
kilo = 1000.
else:
# Officially, a kibibyte
kilo = 1024.
if bytes >= (kilo * kilo * kilo):
return "%.1f GB" % (bytes / (kilo * kilo * kilo))
elif bytes >= 1000000:
return "%.1f MB" % (bytes / (kilo * kilo))
elif bytes >= 1000:
return "%.1f KB" % (bytes / kilo)
else:
return "%.0f bytes" % bytes
def format_period(seconds):
"""
Formats a period in seconds into a string
"""
if seconds == (24 * 60 * 60):
return "day"
elif seconds == (60 * 60):
return "hour"
elif seconds > (24 * 60 * 60):
return "%.1f days" % (seconds / 24. / 60. / 60.)
elif seconds > (60 * 60):
return "%.1f hours" % (seconds / 60. / 60.)
elif seconds > (60):
return "%.1f minutes" % (seconds / 60.)
else:
return "%.0f seconds" % seconds
def slicemail(slice, subject, body):
'''
Front end to sendmail. Sends email to slice alias with given subject and body.
'''
config = Config()
sendmail = os.popen("/usr/sbin/sendmail -N never -t -f%s" % config.PLC_MAIL_SUPPORT_ADDRESS, "w")
# Parsed from MyPLC config
to = [config.PLC_MAIL_MOM_LIST_ADDRESS]
if slice is not None and slice != "root":
to.append(config.PLC_MAIL_SLICE_ADDRESS.replace("SLICE", slice))
header = {'from': "%s Support <%s>" % (config.PLC_NAME,
config.PLC_MAIL_SUPPORT_ADDRESS),
'to': ", ".join(to),
'version': sys.version.split(" ")[0],
'subject': subject}
# Write headers
sendmail.write(
"""
Content-type: text/plain
From: %(from)s
Reply-To: %(from)s
To: %(to)s
X-Mailer: Python/%(version)s
Subject: %(subject)s
""".lstrip() % header)
# Write body
sendmail.write(body)
# Done
sendmail.close()
class Slice:
"""
Stores the last recorded bandwidth parameters of a slice.
xid - slice context/VServer ID
name - slice name
time - beginning of recording period in UNIX seconds
bytes - low bandwidth bytes transmitted at the beginning of the recording period
i2bytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F)
MaxKByte - total volume of data allowed
ThreshKbyte - After thresh, cap node to (maxkbyte - bytes)/(time left in period)
Maxi2KByte - same as MaxKByte, but for i2
Threshi2Kbyte - same as Threshi2KByte, but for i2
MaxRate - max_rate slice attribute.
Maxi2Rate - max_exempt_rate slice attribute.
Share - Used by Sirius to loan min rates
Sharei2 - Used by Sirius to loan min rates for i2
self.emailed - did slice recv email during this recording period
"""
def __init__(self, xid, name, rspec):
self.xid = xid
self.name = name
self.time = 0
self.bytes = 0
self.i2bytes = 0
self.MaxRate = default_MaxRate
self.MinRate = bwlimit.bwmin // 1000
self.Maxi2Rate = default_Maxi2Rate
self.Mini2Rate = bwlimit.bwmin // 1000
self.MaxKByte = default_MaxKByte
self.ThreshKByte = int(.8 * self.MaxKByte)
self.Maxi2KByte = default_Maxi2KByte
self.Threshi2KByte = int(.8 * self.Maxi2KByte)
self.Share = default_Share
self.Sharei2 = default_Share
self.emailed = False
self.capped = False
self.updateSliceTags(rspec)
bwlimit.set(
xid=self.xid, dev=dev_default,
minrate=self.MinRate * 1000,
maxrate=self.MaxRate * 1000,
maxexemptrate=self.Maxi2Rate * 1000,
minexemptrate=self.Mini2Rate * 1000,
share=self.Share)
def __repr__(self):
return self.name
def updateSliceTags(self, rspec):
'''
Use respects from GetSlivers to PLC to populate slice object. Also
do some sanity checking.
'''
# Sanity check plus policy decision for MinRate:
# Minrate cant be greater than 25% of MaxRate or NodeCap.
MinRate = int(rspec.get("net_min_rate", bwlimit.bwmin // 1000))
if MinRate > int(.25 * default_MaxRate):
MinRate = int(.25 * default_MaxRate)
if MinRate != self.MinRate:
self.MinRate = MinRate
logger.log("bwmon: Updating %s: Min Rate = %s" %(self.name, self.MinRate))
MaxRate = int(rspec.get('net_max_rate', default_MaxRate))
if MaxRate != self.MaxRate:
self.MaxRate = MaxRate
logger.log("bwmon: Updating %s: Max Rate = %s" %(self.name, self.MaxRate))
Mini2Rate = int(rspec.get('net_i2_min_rate', bwlimit.bwmin // 1000))
if Mini2Rate != self.Mini2Rate:
self.Mini2Rate = Mini2Rate
logger.log("bwmon: Updating %s: Min i2 Rate = %s" %(self.name, self.Mini2Rate))
Maxi2Rate = int(rspec.get('net_i2_max_rate', default_Maxi2Rate))
if Maxi2Rate != self.Maxi2Rate:
self.Maxi2Rate = Maxi2Rate
logger.log("bwmon: Updating %s: Max i2 Rate = %s" %(self.name, self.Maxi2Rate))
MaxKByte = int(rspec.get('net_max_kbyte', default_MaxKByte))
if MaxKByte != self.MaxKByte:
self.MaxKByte = MaxKByte
logger.log("bwmon: Updating %s: Max KByte lim = %s" %(self.name, self.MaxKByte))
Maxi2KByte = int(rspec.get('net_i2_max_kbyte', default_Maxi2KByte))
if Maxi2KByte != self.Maxi2KByte:
self.Maxi2KByte = Maxi2KByte
logger.log("bwmon: Updating %s: Max i2 KByte = %s" %(self.name, self.Maxi2KByte))
ThreshKByte = int(rspec.get('net_thresh_kbyte', (MaxKByte * .8)))
if ThreshKByte != self.ThreshKByte:
self.ThreshKByte = ThreshKByte
logger.log("bwmon: Updating %s: Thresh KByte = %s" %(self.name, self.ThreshKByte))
Threshi2KByte = int(rspec.get('net_i2_thresh_kbyte', (Maxi2KByte * .8)))
if Threshi2KByte != self.Threshi2KByte:
self.Threshi2KByte = Threshi2KByte
logger.log("bwmon: Updating %s: i2 Thresh KByte = %s" %(self.name, self.Threshi2KByte))
Share = int(rspec.get('net_share', default_Share))
if Share != self.Share:
self.Share = Share
logger.log("bwmon: Updating %s: Net Share = %s" %(self.name, self.Share))
Sharei2 = int(rspec.get('net_i2_share', default_Share))
if Sharei2 != self.Sharei2:
self.Sharei2 = Sharei2
logger.log("bwmon: Updating %s: Net i2 Share = %s" %(self.name, self.i2Share))
def reset(self, runningrates, rspec):
"""
Begin a new recording period. Remove caps by restoring limits
to their default values.
"""
# Cache share for later comparison
self.Share = runningrates.get('share', 1)
# Query Node Manager for max rate overrides
self.updateSliceTags(rspec)
# Reset baseline time
self.time = time.time()
# Reset baseline byte coutns
self.bytes = runningrates.get('usedbytes', 0)
self.i2bytes = runningrates.get('usedi2bytes', 0)
# Reset email
self.emailed = False
# Reset flag
self.capped = False
# Reset rates.
maxrate = self.MaxRate * 1000
minrate = self.MinRate * 1000
maxi2rate = self.Maxi2Rate * 1000
mini2rate = self.Mini2Rate * 1000
if (maxrate != runningrates.get('maxrate', 0)) or \
(minrate != runningrates.get('maxrate', 0)) or \
(maxi2rate != runningrates.get('maxexemptrate', 0)) or \
(mini2rate != runningrates.get('minexemptrate', 0)) or \
(self.Share != runningrates.get('share', 0)):
logger.log("bwmon: %s reset to %s/%s" % \
(self.name,
bwlimit.format_tc_rate(maxrate),
bwlimit.format_tc_rate(maxi2rate)))
bwlimit.set(xid = self.xid, dev = dev_default,
minrate = self.MinRate * 1000,
maxrate = self.MaxRate * 1000,
maxexemptrate = self.Maxi2Rate * 1000,
minexemptrate = self.Mini2Rate * 1000,
share = self.Share)
def notify(self, new_maxrate, new_maxexemptrate, usedbytes, usedi2bytes):
"""
Notify the slice it's being capped.
"""
# Prepare message parameters from the template
message = ""
params = {'slice': self.name, 'hostname': socket.gethostname(),
'since': time.asctime(time.gmtime(self.time)) + " GMT",
'until': time.asctime(time.gmtime(self.time + period)) + " GMT",
'date': time.asctime(time.gmtime()) + " GMT",
'period': format_period(period)}
if new_maxrate != (self.MaxRate * 1000):
# Format template parameters for low bandwidth message
params['class'] = "low bandwidth"
params['bytes'] = format_bytes(usedbytes - self.bytes)
params['limit'] = format_bytes(self.MaxKByte * 1024)
params['new_maxrate'] = bwlimit.format_tc_rate(new_maxrate)
# Cap low bandwidth burst rate
message += template % params
logger.log("bwmon: ** %(slice)s %(class)s capped at %(new_maxrate)s/s " % params)
if new_maxexemptrate != (self.Maxi2Rate * 1000):
# Format template parameters for high bandwidth message
params['class'] = "high bandwidth"
params['bytes'] = format_bytes(usedi2bytes - self.i2bytes)
params['limit'] = format_bytes(self.Maxi2KByte * 1024)
params['new_maxrate'] = bwlimit.format_tc_rate(new_maxexemptrate)
message += template % params
logger.log("bwmon: ** %(slice)s %(class)s capped at %(new_maxrate)s/s " % params)
# Notify slice
if self.emailed == False:
subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params
if DEBUG:
logger.log("bwmon: "+ subject)
logger.log("bwmon: "+ message + (footer % params))
else:
self.emailed = True
logger.log("bwmon: Emailing %s" % self.name)
slicemail(self.name, subject, message + (footer % params))
def update(self, runningrates, rspec):
"""
Update byte counts and check if byte thresholds have been
exceeded. If exceeded, cap to remaining bytes in limit over remaining time in period.
Recalculate every time module runs.
"""
# cache share for later comparison
runningrates['share'] = self.Share
# Query Node Manager for max rate overrides
self.updateSliceTags(rspec)
usedbytes = runningrates['usedbytes']
usedi2bytes = runningrates['usedi2bytes']
# Check limits.
if usedbytes >= (self.bytes + (self.ThreshKByte * 1024)):
sum = self.bytes + (self.ThreshKByte * 1024)
maxbyte = self.MaxKByte * 1024
bytesused = usedbytes - self.bytes
timeused = int(time.time() - self.time)
# Calcuate new rate. in bit/s
new_maxrate = int(((maxbyte - bytesused) * 8)
/ (period - timeused))
# Never go under MinRate
if new_maxrate < (self.MinRate * 1000):
new_maxrate = self.MinRate * 1000
# State information. I'm capped.
self.capped += True
else:
# Sanity Check
new_maxrate = self.MaxRate * 1000
self.capped += False
if usedi2bytes >= (self.i2bytes + (self.Threshi2KByte * 1024)):
maxi2byte = self.Maxi2KByte * 1024
i2bytesused = usedi2bytes - self.i2bytes
timeused = int(time.time() - self.time)
# Calcuate New Rate.
new_maxi2rate = int(((maxi2byte - i2bytesused) * 8)
/(period - timeused))
# Never go under MinRate
if new_maxi2rate < (self.Mini2Rate * 1000):
new_maxi2rate = self.Mini2Rate * 1000
# State information. I'm capped.
self.capped += True
else:
# Sanity
new_maxi2rate = self.Maxi2Rate * 1000
self.capped += False
# Check running values against newly calculated values so as not to run tc
# unnecessarily
if (runningrates['maxrate'] != new_maxrate) or \
(runningrates['minrate'] != self.MinRate * 1000) or \
(runningrates['maxexemptrate'] != new_maxi2rate) or \
('minexemptrate' in runningrates and runningrates['minexemptrate'] != self.Mini2Rate * 1000) or \
(runningrates['share'] != self.Share):
# Apply parameters
bwlimit.set(xid = self.xid, dev = dev_default,
minrate = self.MinRate * 1000,
maxrate = new_maxrate,
minexemptrate = self.Mini2Rate * 1000,
maxexemptrate = new_maxi2rate,
share = self.Share)
# Notify slice
if self.capped == True:
self.notify(new_maxrate, new_maxi2rate, usedbytes, usedi2bytes)
def gethtbs(root_xid, default_xid):
"""
Return dict {xid: {*rates}} of running htbs as reported by tc that have names.
Turn off HTBs without names.
"""
livehtbs = {}
for params in bwlimit.get(dev = dev_default):
(xid, share,
minrate, maxrate,
minexemptrate, maxexemptrate,
usedbytes, usedi2bytes) = params
name = bwlimit.get_slice(xid)
if (name is None) \
and (xid != root_xid) \
and (xid != default_xid):
# Orphaned (not associated with a slice) class
name = "%d?" % xid
logger.log("bwmon: Found orphaned HTB %s. Removing." %name)
bwlimit.off(xid, dev = dev_default)
livehtbs[xid] = {'share': share,
'minrate': minrate,
'maxrate': maxrate,
'maxexemptrate': maxexemptrate,
'minexemptrate': minexemptrate,
'usedbytes': usedbytes,
'name': name,
'usedi2bytes': usedi2bytes}
return livehtbs
def sync(nmdbcopy):
"""
Syncs tc, db, and bwmon.pickle.
Then, starts new slices, kills old ones, and updates byte accounts for each running slice.
Sends emails and caps those that went over their limit.
"""
# Defaults
global DB_FILE, \
period, \
default_MaxRate, \
default_Maxi2Rate, \
default_MaxKByte, \
default_Maxi2KByte, \
default_Share, \
dev_default
# All slices
names = []
# In case the limits have changed.
default_MaxRate = int(bwlimit.get_bwcap(dev_default) / 1000)
default_Maxi2Rate = int(bwlimit.bwmax / 1000)
# Incase default isn't set yet.
if default_MaxRate == -1:
default_MaxRate = 1000000
# xxx $Id$
# with svn we used to have a trick to detect upgrades of this file
# this has gone with the move to git, without any noticeable effect on operations though
try:
f = open(DB_FILE, "r+")
logger.verbose("bwmon: Loading %s" % DB_FILE)
(version, slices, deaddb) = pickle.load(f)
f.close()
# Check version of data file
if version != "$Id$":
logger.log("bwmon: Not using old version '%s' data file %s" % (version, DB_FILE))
raise Exception
except Exception:
version = "$Id$"
slices = {}
deaddb = {}
# Get/set special slice IDs
root_xid = bwlimit.get_xid("root")
default_xid = bwlimit.get_xid("default")
# Since root is required for sanity, its not in the API/plc database, so pass {}
# to use defaults.
if root_xid not in list(slices.keys()):
slices[root_xid] = Slice(root_xid, "root", {})
slices[root_xid].reset({}, {})
# Used by bwlimit. pass {} since there is no rspec (like above).
if default_xid not in list(slices.keys()):
slices[default_xid] = Slice(default_xid, "default", {})
slices[default_xid].reset({}, {})
live = {}
# Get running slivers that should be on this node (from plc). {xid: name}
# db keys on name, bwmon keys on xid. db doesnt have xid either.
for plcSliver in list(nmdbcopy.keys()):
live[bwlimit.get_xid(plcSliver)] = nmdbcopy[plcSliver]
logger.verbose("bwmon: Found %s instantiated slices" % list(live.keys()).__len__())
logger.verbose("bwmon: Found %s slices in dat file" % list(slices.values()).__len__())
# Get actual running values from tc.
# Update slice totals and bandwidth. {xid: {values}}
kernelhtbs = gethtbs(root_xid, default_xid)
logger.verbose("bwmon: Found %s running HTBs" % list(kernelhtbs.keys()).__len__())
# The dat file has HTBs for slices, but the HTBs aren't running
nohtbslices = set(slices.keys()) - set(kernelhtbs.keys())
logger.verbose( "bwmon: Found %s slices in dat but not running." % nohtbslices.__len__())
# Reset tc counts.
for nohtbslice in nohtbslices:
if nohtbslice in live:
slices[nohtbslice].reset( {}, live[nohtbslice]['_rspec'] )
else:
logger.log("bwmon: Removing abondoned slice %s from dat." % nohtbslice)
del slices[nohtbslice]
# The dat file doesnt have HTB for the slice but kern has HTB
slicesnodat = set(kernelhtbs.keys()) - set(slices.keys())
logger.verbose( "bwmon: Found %s slices with HTBs but not in dat" % slicesnodat.__len__())
for slicenodat in slicesnodat:
# But slice is running
if slicenodat in live:
# init the slice. which means start accounting over since kernel
# htb was already there.
slices[slicenodat] = Slice(slicenodat,
live[slicenodat]['name'],
live[slicenodat]['_rspec'])
# Get new slices.
# Slices in GetSlivers but not running HTBs
newslicesxids = set(live.keys()) - set(kernelhtbs.keys())
logger.verbose("bwmon: Found %s new slices" % newslicesxids.__len__())
# Setup new slices
for newslice in newslicesxids:
# Delegated slices dont have xids (which are uids) since they haven't been
# instantiated yet.
if newslice != None and ('_rspec' in live[newslice]) == True:
# Check to see if we recently deleted this slice.
if live[newslice]['name'] not in list(deaddb.keys()):
logger.log( "bwmon: new slice %s" % live[newslice]['name'] )
# _rspec is the computed rspec: NM retrieved data from PLC, computed loans
# and made a dict of computed values.
slices[newslice] = Slice(newslice, live[newslice]['name'], live[newslice]['_rspec'])
slices[newslice].reset( {}, live[newslice]['_rspec'] )
# Double check time for dead slice in deaddb is within 24hr recording period.
elif (time.time() <= (deaddb[live[newslice]['name']]['slice'].time + period)):
deadslice = deaddb[live[newslice]['name']]
logger.log("bwmon: Reinstantiating deleted slice %s" % live[newslice]['name'])
slices[newslice] = deadslice['slice']
slices[newslice].xid = newslice
# Start the HTB
newvals = {"maxrate": deadslice['slice'].MaxRate * 1000,
"minrate": deadslice['slice'].MinRate * 1000,
"maxexemptrate": deadslice['slice'].Maxi2Rate * 1000,
"usedbytes": deadslice['htb']['usedbytes'] * 1000,
"usedi2bytes": deadslice['htb']['usedi2bytes'],
"share":deadslice['htb']['share']}
slices[newslice].reset(newvals, live[newslice]['_rspec'])
# Bring up to date
slices[newslice].update(newvals, live[newslice]['_rspec'])
# Since the slice has been reinitialed, remove from dead database.
del deaddb[deadslice['slice'].name]
del newvals
else:
logger.log("bwmon: Slice %s doesn't have xid. Skipping." % live[newslice]['name'])
# Move dead slices that exist in the pickle file, but
# aren't instantiated by PLC into the dead dict until
# recording period is over. This is to avoid the case where a slice is dynamically created
# and destroyed then recreated to get around byte limits.
deadxids = set(slices.keys()) - set(live.keys())
logger.verbose("bwmon: Found %s dead slices" % (deadxids.__len__() - 2))
for deadxid in deadxids:
if deadxid == root_xid or deadxid == default_xid:
continue
logger.log("bwmon: removing dead slice %s " % deadxid)
if deadxid in slices and deadxid in kernelhtbs:
# add slice (by name) to deaddb
logger.log("bwmon: Saving bandwidth totals for %s." % slices[deadxid].name)
deaddb[slices[deadxid].name] = {'slice': slices[deadxid], 'htb': kernelhtbs[deadxid]}
del slices[deadxid]
if deadxid in kernelhtbs:
logger.verbose("bwmon: Removing HTB for %s." % deadxid)
bwlimit.off(deadxid, dev = dev_default)
# Clean up deaddb
for deadslice in list(deaddb.keys()):
if (time.time() >= (deaddb[deadslice]['slice'].time + period)):
logger.log("bwmon: Removing dead slice %s from dat." \
% deaddb[deadslice]['slice'].name)
del deaddb[deadslice]
# Get actual running values from tc since we've added and removed buckets.
# Update slice totals and bandwidth. {xid: {values}}
kernelhtbs = gethtbs(root_xid, default_xid)
logger.verbose("bwmon: now %s running HTBs" % list(kernelhtbs.keys()).__len__())
# Update all byte limites on all slices
for (xid, slice) in slices.items():
# Monitor only the specified slices
if xid == root_xid or xid == default_xid: continue
if names and name not in names:
continue
if (time.time() >= (slice.time + period)) or \
(kernelhtbs[xid]['usedbytes'] < slice.bytes) or \
(kernelhtbs[xid]['usedi2bytes'] < slice.i2bytes):
# Reset to defaults every 24 hours or if it appears
# that the byte counters have overflowed (or, more
# likely, the node was restarted or the HTB buckets
# were re-initialized).
slice.reset(kernelhtbs[xid], live[xid]['_rspec'])
elif ENABLE:
logger.verbose("bwmon: Updating slice %s" % slice.name)
# Update byte counts
slice.update(kernelhtbs[xid], live[xid]['_rspec'])
logger.verbose("bwmon: Saving %s slices in %s" % (list(slices.keys()).__len__(), DB_FILE))
f = open(DB_FILE, "w")
pickle.dump((version, slices, deaddb), f)
f.close()
# doesnt use generic default interface because this runs as its own thread.
# changing the config variable will not have an effect since GetSlivers: pass
def getDefaults(nmdbcopy):
'''
Get defaults from default slice's slice attributes.
'''
status = True
# default slice
dfltslice = nmdbcopy.get(Config().PLC_SLICE_PREFIX+"_default")
if dfltslice:
if dfltslice['rspec']['net_max_rate'] == -1:
allOff()
status = False
return status
def allOff():
"""
Turn off all slice HTBs
"""
# Get/set special slice IDs
root_xid = bwlimit.get_xid("root")
default_xid = bwlimit.get_xid("default")
kernelhtbs = gethtbs(root_xid, default_xid)
if len(kernelhtbs):
logger.log("bwmon: Disabling all running HTBs.")
for htb in list(kernelhtbs.keys()): bwlimit.off(htb, dev = dev_default)
lock = threading.Event()
def run():
"""
When run as a thread, wait for event, lock db, deep copy it, release it,
run bwmon.GetSlivers(), then go back to waiting.
"""
logger.verbose("bwmon: Thread started")
while True:
lock.wait()
logger.verbose("bwmon: Event received. Running.")
database.db_lock.acquire()
nmdbcopy = copy.deepcopy(database.db)
database.db_lock.release()
try:
if getDefaults(nmdbcopy) and len(bwlimit.tc("class show dev %s" % dev_default)) > 0:
# class show to check if net:InitNodeLimit:bwlimit.init has run.
sync(nmdbcopy)
else: logger.log("bwmon: BW limits DISABLED.")
except: logger.log_exc("bwmon failed")
lock.clear()
def start(*args):
tools.as_daemon_thread(run)
def GetSlivers(*args):
logger.verbose ("bwmon: triggering dummy GetSlivers")
pass