def __init__(self, config, name):
# Guard the data with a semaphore to ensure consistency.
self.data_sem = threading.Semaphore()
self.properties = {}
self.statistics = deque()
self.variables = {}
self.name = name
self.fields = None
self.optional_fields = None
self.collectors = []
self.logger = logging.getLogger('mom.Monitor')
plot_dir = config.get('__int__', 'plot-subdir')
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
else:
self.plotter = None
graphite_host = config.get('main', 'graphite-host')
graphite_port = config.get('main', 'graphite-port')
graphite_protocol = config.get('main', 'graphite-protocol')
if graphite_host != '':
self.graphite = Graphite(graphite_host, int(graphite_port),
graphite_protocol, name)
else:
self.graphite = None
self.ready = None
self._terminate = False
def set_policy(self, total_mem, plot_dir):
self.total_mem_ratio = total_mem
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields)
else:
self.plotter = None
class RPPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.RPPolicy')
self.policy_sem = threading.Semaphore()
self.start = time.time()
#self.fields = set(['Used', 'User', 'System', 'Nice', 'Idle', 'IOwait', 'Irq', 'Softirq', 'Total', 'Pagefault', 'Swapin', 'Swapout'])
self.fields = set(['Used', 'User', 'System', 'Nice', 'Idle', 'IOwait', 'Irq', 'Softirq', 'Total', 'Pagefault'])
self.VM_Infos = {}
def set_policy(self, type, total_mem, plot_dir, alpha, beta):
self.type = type
self.total_mem_ratio = total_mem
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields)
else:
self.plotter = None
def getMem(self):
Max = 1500000
Min = 930000
now = time.time() - self.start
if now < 300:
return now*(Min-Max)/300.0 + Max
elif now < 360:
return Min
elif now < 660:
return now*(Max-Min)/300.0 + 2.2*Min - 1.2*Max
def evaluate(self, host, guest_list):
for guest in guest_list:
name = guest.Prop('name')
if name not in self.VM_Infos or self.VM_Infos[name] is None:
info = VM_Info(name)
info.initAttribute(guest)
self.VM_Infos[name] = info
else:
info = self.VM_Infos[name]
info.update(guest)
target = self.getMem()
for name, info in self.VM_Infos.iteritems():
info.setAttribute('Allocated', target)
info.setBalloon()
if self.plotter is not None:
for name, info in self.VM_Infos.iteritems():
data = info.getData(self.fields)
self.plotter.plot(data)
def set_policy(self, type, total_mem, plot_dir, alpha, beta):
self.type = type
self.total_mem_ratio = total_mem
self.alpha = alpha
self.beta = beta
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields.union(self.fit_fields))
else:
self.plotter = None
def __init__(self, config, name):
# Guard the data with a lock to ensure consistency.
self.data_lock = threading.Lock()
self.properties = {}
self.statistics = deque()
self.variables = {}
self.name = name
self.fields = None
self.optional_fields = None
self.collectors = []
self.logger = logging.getLogger('mom.Monitor')
plot_dir = config.get('__int__', 'plot-subdir')
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
else:
self.plotter = None
self.ready = None
self._terminate = False
class RPPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.RPPolicy')
self.policy_sem = threading.Semaphore()
self.start = True
#self.fields = set(['Used', 'User', 'System', 'Nice', 'Idle', 'IOwait', 'Irq', 'Softirq', 'Total', 'Pagefault', 'Swapin', 'Swapout'])
self.fields = set([
'Used', 'User', 'System', 'Nice', 'Idle', 'IOwait', 'Irq',
'Softirq', 'Total', 'Pagefault'
])
self.VM_Infos = {}
def set_policy(self, type, total_mem, plot_dir, alpha, beta):
self.type = type
self.total_mem_ratio = total_mem
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields)
else:
self.plotter = None
def evaluate(self, host, guest_list):
for guest in guest_list:
name = guest.Prop('name')
if name not in self.VM_Infos or self.VM_Infos[name] is None:
info = VM_Info(name)
info.initAttribute(guest)
self.VM_Infos[name] = info
else:
info = self.VM_Infos[name]
info.update(guest)
if self.plotter is not None:
for name, info in self.VM_Infos.iteritems():
data = info.getData(self.fields)
self.plotter.plot(data)
def __init__(self, config, name):
# Guard the data with a semaphore to ensure consistency.
self.data_sem = threading.Semaphore()
self.properties = {}
self.statistics = deque()
self.variables = {}
self.name = name
self.fields = None
self.collectors = []
self.logger = logging.getLogger('mom.Monitor')
plot_dir = config.get('__int__', 'plot-subdir')
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
else:
self.plotter = None
self.ready = None
self._terminate = False
class RPPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.RPPolicy')
self.policy_sem = threading.Semaphore()
self.start = True
self.fields = set([
'VM', 'Used', 'Swap', 'Worksize', 'Balloon', 'User', 'Start-time',
'IOwait', 'Softirq', 'System', 'Total', 'Speed', 'Rate',
'Allocated', 'Pagefault'
])
self.fit_fields = set([
'users', 'standard_users', 'ave_users', 'pfs', 'ave_pfs',
'ins_speed'
])
self.VM_Infos = {}
self.pre_even = {}
self.pre_odd = {}
self.count = 0
self.pre_state = 0
def set_policy(self, type, total_mem, plot_dir, alpha, beta):
self.type = type
self.total_mem_ratio = total_mem
self.alpha = alpha
self.beta = beta
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields.union(self.fit_fields))
else:
self.plotter = None
def firstTime(self, total_mem):
num = len(self.VM_Infos)
share = total_mem / num
for name, info in self.VM_Infos.iteritems():
info.setAttribute('Allocated', share)
info.setBalloon()
def pre_allocate(self, total_mem):
if len(self.pre_even) == 0:
return
min_mem = self.VM_Infos.values()[0].getAttribute('Min')
#share_odd = (total_mem - min_mem)*4/(WINDOW + AVE_LEN)
share_odd = Min_share
share_even = -(share_odd * len(self.pre_odd) / len(self.pre_even))
self.logger.info(
"total_mem: %s, min_mem: %s, share_odd: %s, share_even: %s",
total_mem, min_mem, share_odd, share_even)
if self.pre_state == 0:
print(self.pre_even.values()[0].getAttribute('Allocated') +
share_even)
print(min_mem)
if self.pre_odd.values()[0].getAttribute(
'Allocated') - share_odd < min_mem:
self.pre_state = 1
else:
if self.pre_even.values()[0].getAttribute(
'Allocated') + share_even < min_mem:
self.pre_state = 0
self.logger.info("count: %s, pre_state: %s", self.count,
self.pre_state)
if self.pre_state == 0:
share_odd = -share_odd
share_even = -share_even
for name, info in self.pre_odd.iteritems():
info.addAttribute('Allocated', share_odd)
info.setBalloon()
for name, info in self.pre_even.iteritems():
info.addAttribute('Allocated', share_even)
info.setBalloon()
def dynamic_allocate(self):
ranks = {}
for name, info in self.VM_Infos.iteritems():
rate = info.getAttribute('Rate')
if rate > 0:
ranks[info] = rate
rank_l = sorted(ranks.items(), lambda x, y: cmp(x[1], y[1]))
max_info = None
max_share = 0
max_index = 0
min_info = None
min_share = 0
min_index = 0
for i in range(0, len(rank_l)):
info = rank_l[i][0]
if info.dist2high() > 0:
min_info = info
min_share = info.dist2high()
min_index = i
break
for i in range(len(rank_l) - 1, -1, -1):
info = rank_l[i][0]
if info.dist2low() > 0:
max_info = info
max_share = info.dist2low()
max_index = i
break
if min_index >= max_index:
return
share = min(max_share, min_share, Min_share)
#self.logger.info("Giver:%s, Taker:%s, Share:%s", max_info.name, min_info.name, share)
max_info.addAttribute('Allocated', -share)
min_info.addAttribute('Allocated', share)
for name, info in self.VM_Infos.iteritems():
info.setBalloon()
def meanByFree(self, active, inactive):
total = 0
for info in inactive:
share = min(info.getAttribute('Free') - Max_free, \
info.getAttribute('Allocated') - info.getAttribute('Min'))
if share <= 0:
continue
info.addAttribute('Allocated', -share)
total += share
ave_share = total / len(active)
for info in active:
info.addAttribute('Allocated', ave_share)
for name, info in self.VM_Infos.iteritems():
info.setBalloon()
def check(self):
active = []
inactive = []
for name, info in self.VM_Infos.iteritems():
if info.getAttribute('Free') < Max_free:
active.append(info)
else:
inactive.append(info)
# self.logger.info("active list:%s", active)
# self.logger.info("inactive list:%s", inactive)
if len(active) == 0:
return False
elif len(inactive) == 0:
return True
else:
self.meanByFree(active, inactive)
return False
def evaluate(self, host, guest_list):
total_mem = host.mem_available * float(self.total_mem_ratio)
for guest in guest_list:
name = guest.Prop('name')
if name not in self.VM_Infos or self.VM_Infos[name] is None:
info = VM_Info(name, self.alpha, self.beta)
info.initAttribute(guest)
self.VM_Infos[name] = info
else:
info = self.VM_Infos[name]
info.update(guest)
if self.start:
self.count += 1
self.firstTime(total_mem)
if self.count >= AVE_LEN:
self.start = False
return
if self.check():
self.dynamic_allocate()
if self.plotter is not None:
for name, info in self.VM_Infos.iteritems():
data = info.getData(self.fields, self.fit_fields)
self.plotter.plot(data)
class Monitor(object):
"""
The Monitor class represents an entity, about which, data is collected and
reported. Each monitor has a dictionary of properties which are relatively
static such as a name or ID. Additionally, statistics are collected over
time and queued so averages and trends can be analyzed.
"""
def __init__(self, config, name):
# Guard the data with a semaphore to ensure consistency.
self.data_sem = threading.Semaphore()
self.properties = {}
self.statistics = deque()
self.variables = {}
self.name = name
self.fields = None
self.optional_fields = None
self.collectors = []
self.logger = logging.getLogger('mom.Monitor')
plot_dir = config.get('__int__', 'plot-subdir')
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
else:
self.plotter = None
self.ready = None
self._terminate = False
@property
def valid_fields(self):
return self.fields.union(self.optional_fields)
def collect(self):
"""
Collect a set of statistics by invoking all defined collectors and
merging the data into one dictionary and pushing it onto the deque of
historical statistics. Maintain a history length as specified in the
config file.
Note: Priority is given to collectors based on the order that they are
listed in the config file (ie. if two collectors produce the same
statistic only the value produced by the first collector will be saved).
Return: The dictionary of collected statistics
"""
# The first time we are called, populate the list of expected fields
if self.fields is None:
self.fields = set()
for c in self.collectors:
self.fields |= c.getFields()
self.logger.debug("Using fields: %s", repr(self.fields))
# The first time we are called, populate the list of optional fields
if self.optional_fields is None:
self.optional_fields = set()
for c in self.collectors:
self.optional_fields |= c.getOptionalFields()
self.logger.debug("Using optional fields: %s", repr(self.optional_fields))
# Remove mandatory fields from the optional list
# This can happen when more than one collector is able to provide
# the value
self.optional_fields = self.optional_fields.difference(self.fields)
if self.plotter is not None:
self.plotter.setFields(self.fields.union(self.optional_fields))
data = {}
for c in self.collectors:
try:
collected = c.collect()
if collected is None:
self.logger.debug("Collector %s did not "
"return any data", str(c))
continue
for (key, val) in collected.items():
if key not in data or data[key] is None:
data[key] = val
except Collector.CollectionError, e:
self._disp_collection_error("Collection error: %s" % e.msg)
except Collector.FatalError, e:
self._set_not_ready("Fatal Collector error: %s" % e.msg)
self.terminate()
return None
except Exception:
self.logger.exception("Unexpected collection error")
class Monitor(object):
"""
The Monitor class represents an entity, about which, data is collected and
reported. Each monitor has a dictionary of properties which are relatively
static such as a name or ID. Additionally, statistics are collected over
time and queued so averages and trends can be analyzed.
"""
def __init__(self, config, name):
# Guard the data with a semaphore to ensure consistency.
self.data_sem = threading.Semaphore()
self.properties = {}
self.statistics = deque()
self.variables = {}
self.name = name
self.fields = None
self.optional_fields = None
self.collectors = []
self.logger = logging.getLogger('mom.Monitor')
plot_dir = config.get('__int__', 'plot-subdir')
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
else:
self.plotter = None
self.ready = None
self._terminate = False
@property
def valid_fields(self):
return self.fields.union(self.optional_fields)
def collect(self):
"""
Collect a set of statistics by invoking all defined collectors and
merging the data into one dictionary and pushing it onto the deque of
historical statistics. Maintain a history length as specified in the
config file.
Note: Priority is given to collectors based on the order that they are
listed in the config file (ie. if two collectors produce the same
statistic only the value produced by the first collector will be saved).
Return: The dictionary of collected statistics
"""
# The first time we are called, populate the list of expected fields
if self.fields is None:
self.fields = set()
for c in self.collectors:
self.fields |= c.getFields()
self.logger.debug("Using fields: %s", repr(self.fields))
# The first time we are called, populate the list of optional fields
if self.optional_fields is None:
self.optional_fields = set()
for c in self.collectors:
self.optional_fields |= c.getOptionalFields()
self.logger.debug("Using optional fields: %s",
repr(self.optional_fields))
# Remove mandatory fields from the optional list
# This can happen when more than one collector is able to provide
# the value
self.optional_fields = self.optional_fields.difference(self.fields)
if self.plotter is not None:
self.plotter.setFields(self.fields.union(self.optional_fields))
data = {}
for c in self.collectors:
try:
collected = c.collect()
if collected is None:
self.logger.debug(
"Collector %s did not "
"return any data", str(c))
continue
for (key, val) in collected.items():
if key not in data or data[key] is None:
data[key] = val
except Collector.CollectionError as e:
self._disp_collection_error("Collection error: %s" % e.msg)
except Collector.FatalError as e:
self._set_not_ready("Fatal Collector error: %s" % e.msg)
self.terminate()
return None
except Exception:
self.logger.exception("Unexpected collection error")
if not set(data).issuperset(self.fields):
self._set_not_ready("Incomplete data: missing %s" % \
(self.fields - set(data)))
return None
# put None to all unset (optional) fields
for k in self.optional_fields:
data.setdefault(k, None)
self.data_sem.acquire()
self.statistics.append(data)
if len(self.statistics) > self.config.getint('main',
'sample-history-length'):
self.statistics.popleft()
self.data_sem.release()
self._set_ready()
if self.plotter is not None:
self.plotter.plot(data)
return data
def interrogate(self):
"""
Take a snapshot of this Monitor object and return an Entity object which
is useful for rules processing.
Return: A new Entity object
"""
if self.ready is not True:
return None
ret = Entity(monitor=self)
self.data_sem.acquire()
for prop in self.properties.keys():
ret._set_property(prop, self.properties[prop])
for var in self.variables.keys():
ret._set_variable(var, self.variables[var])
ret._set_statistics(self.statistics)
self.data_sem.release()
ret._finalize()
return ret
def update_variables(self, variables):
"""
Update the variables array to store any updates from an Entity
"""
self.data_sem.acquire()
for (var, val) in variables.items():
self.variables[var] = val
self.data_sem.release()
def terminate(self):
"""
Instruct the Monitor to shut down
"""
self._terminate = True
def isReady(self):
"""
Check if all configured Collectors are working properly.
"""
return bool(self.ready)
def _set_ready(self):
if self.ready is not True:
self.logger.info('%s is ready', self.name)
self.ready = True
def _disp_collection_error(self, message=None):
if message is not None:
if self.ready is False:
self.logger.debug('%s: %s', self.name, message)
else: # True or None
self.logger.warn('%s: %s', self.name, message)
def _set_not_ready(self, message=None):
self.ready = False
self._disp_collection_error(message)
def should_run(self):
"""
Helper to determine if the Monitor should continue to run.
"""
return (self.config.getint('__int__', 'running') == 1
and not self._terminate)
class RPPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.RPPolicy')
self.policy_sem = threading.Semaphore()
self.start = True
self.fields = set([
'VM', 'Used', 'Swap', 'Worksize', 'Balloon', 'User', 'Start-time',
'IOwait', 'Softirq', 'System', 'Total', 'Speed', 'Rate',
'Allocated', 'Pagefault'
])
self.fit_fields = set([
'users', 'standard_users', 'ave_users', 'pfs', 'ave_pfs',
'ins_speed'
])
self.VM_Infos = {}
self.count = 0
self.givers = {}
self.takers = {}
def set_policy(self, type, total_mem, plot_dir, alpha, beta):
self.type = type
self.total_mem_ratio = total_mem
self.alpha = alpha
self.beta = beta
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields.union(self.fit_fields))
else:
self.plotter = None
def firstTime(self, total_mem):
num = len(self.VM_Infos)
share = total_mem / num
for name, info in self.VM_Infos.iteritems():
info.setAttribute('Allocated', share)
info.setBalloon()
def dynamic_allocate(self):
ranks = {}
for name, info in self.VM_Infos.iteritems():
rate = info.getAttribute('Rate')
if rate > 0:
ranks[info] = rate
rank_l = sorted(ranks.items(), lambda x, y: cmp(x[1], y[1]))
N = len(rank_l)
if N < 2:
return
index_g = N - 1
giver = rank_l[index_g][0]
giver_blk = giver.dist2low()
while giver_blk <= 0 and index_g > 0:
index_g -= 1
giver = rank_l[index_g][0]
giver_blk = giver.dist2low()
taker = rank_l[0][0]
taker_blk = taker.dist2high()
if index_g > 0:
final_blk = min(giver_blk, taker_blk, Min_share)
self.givers[giver] = -final_blk
self.takers[taker] = final_blk
def meanByFree(self):
total_free = 0
frees = {}
for name, info in self.VM_Infos.iteritems():
if info.getAttribute('Used') < info.getAttribute('Min'):
frees[name] = info.getAttribute(
'Allocated') - info.getAttribute('Min')
else:
frees[name] = info.getAttribute('Free')
total_free += frees[name]
target = total_free / len(self.VM_Infos)
for name, info in self.VM_Infos.iteritems():
free = frees[name]
blk = target - free
if blk < 0:
self.givers[info] = blk
else:
self.takers[info] = blk
def seperate(self):
busy, idle = [], []
for name, info in self.VM_Infos.iteritems():
if info.getAttribute('Free') < Max_free:
busy.append(info)
else:
idle.append(info)
return busy, idle
def trigger_balloon(self, maps):
for info, blk in maps.iteritems():
info.addAttribute('Allocated', blk)
info.setBalloon()
def check(self, total_mem):
total = 0
for name, info in self.VM_Infos.iteritems():
total += info.getAttribute('Allocated')
self.logger.info("Total Allocated: %s, Distance : %s", total,
total - total_mem)
def evaluate(self, host, guest_list):
self.givers, self.takers = {}, {}
total_mem = host.mem_available * float(self.total_mem_ratio)
for guest in guest_list:
name = guest.Prop('name')
if name not in self.VM_Infos or self.VM_Infos[name] is None:
info = VM_Info(name, self.alpha, self.beta)
info.initAttribute(guest)
self.VM_Infos[name] = info
else:
info = self.VM_Infos[name]
info.update(guest)
if self.start:
self.count += 1
self.firstTime(total_mem)
if self.count >= AVE_LEN:
self.start = False
return
busy_vm, idle_vm = self.seperate()
if len(idle_vm) == 0:
self.dynamic_allocate()
elif len(busy_vm) > 0:
self.meanByFree()
self.check(total_mem)
self.trigger_balloon(self.givers)
self.trigger_balloon(self.takers)
if self.plotter is not None:
for name, info in self.VM_Infos.iteritems():
data = info.getData(self.fields, self.fit_fields)
self.plotter.plot(data)
class Monitor(object):
"""
The Monitor class represents an entity, about which, data is collected and
reported. Each monitor has a dictionary of properties which are relatively
static such as a name or ID. Additionally, statistics are collected over
time and queued so averages and trends can be analyzed.
"""
def __init__(self, config, name):
# Guard the data with a semaphore to ensure consistency.
self.data_sem = threading.Semaphore()
self.properties = {}
self.statistics = deque()
self.variables = {}
self.name = name
self.fields = None
self.optional_fields = None
self.collectors = []
self.logger = logging.getLogger('mom.Monitor')
plot_dir = config.get('__int__', 'plot-subdir')
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
else:
self.plotter = None
self.ready = None
self._terminate = False
@property
def valid_fields(self):
return self.fields.union(self.optional_fields)
def collect(self):
"""
Collect a set of statistics by invoking all defined collectors and
merging the data into one dictionary and pushing it onto the deque of
historical statistics. Maintain a history length as specified in the
config file.
Note: Priority is given to collectors based on the order that they are
listed in the config file (ie. if two collectors produce the same
statistic only the value produced by the first collector will be saved).
Return: The dictionary of collected statistics
"""
# The first time we are called, populate the list of expected fields
if self.fields is None:
self.fields = set()
for c in self.collectors:
self.fields |= c.getFields()
self.logger.debug("Using fields: %s", repr(self.fields))
# The first time we are called, populate the list of optional fields
if self.optional_fields is None:
self.optional_fields = set()
for c in self.collectors:
self.optional_fields |= c.getOptionalFields()
self.logger.debug("Using optional fields: %s",
repr(self.optional_fields))
# Remove mandatory fields from the optional list
# This can happen when more than one collector is able to provide
# the value
self.optional_fields = self.optional_fields.difference(self.fields)
if self.plotter is not None:
self.plotter.setFields(self.fields.union(self.optional_fields))
data = {}
for c in self.collectors:
try:
collected = c.collect()
if collected is None:
self.logger.debug(
"Collector %s did not "
"return any data", str(c))
continue
for (key, val) in collected.items():
if key not in data or data[key] is None:
data[key] = val
except Collector.CollectionError, e:
self._disp_collection_error("Collection error: %s" % e.msg)
except Collector.FatalError, e:
self._set_not_ready("Fatal Collector error: %s" % e.msg)
self.terminate()
return None
except Exception:
self.logger.exception("Unexpected collection error")
class WFMPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.Policy')
self.policy_sem = threading.Semaphore()
self.fields = set([
'VM', 'Used', 'Swap', 'Worksize', 'Balloon', 'User', 'Start-time',
'IOwait', 'Softirq', 'System', 'Total', 'Allocated', 'Pagefault'
])
self.total_used = {}
self.VM_Infos = {}
def _validate_config(self, host, guest_list):
for id, guest in guest_list:
total = guest.GetVar('mem_available')
if total <= self.vmpres_threshold:
self.logger.error("invalid 'vmpres_threshold'")
return False
total = host.GetVar('mem_available')
if total <= self.hostpres_threshold:
self.logger.error("invalid 'hostpres_threshold'")
return False
return True
def set_policy(self, total_mem, plot_dir):
self.total_mem_ratio = total_mem
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields)
else:
self.plotter = None
def setBalloon(self, vm, target):
vm.Control('balloon_target', target)
def evaluate(self, host, guest_list):
total_mem = host.mem_available * float(self.total_mem_ratio)
demands = {}
allocation = {}
wei_allo = {}
for guest in guest_list:
name = guest.Prop('name')
if name not in self.VM_Infos or self.VM_Infos[name] is None:
info = VM_Info(name)
info.initAttribute(guest)
self.VM_Infos[name] = info
else:
info = self.VM_Infos[name]
info.update(guest)
if name not in self.total_used or self.total_used[name] is None:
self.total_used[name] = 0
allocation[name] = MinLoc
total_mem -= MinLoc
demands[name] = info.getAttribute('Worksize')
self.total_used[name] += info.getAttribute('Used') - 150000
wei_allo[name] = self.total_used[name] / info.getAttribute(
'Weight')
rank_l = sorted(wei_allo.items(), lambda x, y: cmp(x[1], y[1]))
print rank_l
index = 0
while total_mem > 0 and index < len(rank_l):
name = rank_l[index][0]
demand = demands[name]
index += 1
if demand > MinLoc:
demand = demand - MinLoc
if demand > total_mem:
allocation[name] += total_mem
total_mem = 0
else:
total_mem -= demand
allocation[name] += demand
for name, alloc in allocation.iteritems():
info = self.VM_Infos[name]
info.setAttribute('Allocated', alloc)
for name, info in self.VM_Infos.iteritems():
info.setBalloon()
if self.plotter is not None:
for name, info in self.VM_Infos.iteritems():
data = info.getData(self.fields)
self.plotter.plot(data)
class RPPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.RPPolicy')
self.policy_sem = threading.Semaphore()
self.start = True
self.fields = set([
'VM', 'Used', 'Swap', 'Worksize', 'Balloon', 'User', 'Start-time',
'Total', 'Speed', 'Rate', 'Allocated', 'Pagefault'
])
self.VM_Infos = {}
def set_policy(self, type, total_mem, plot_dir):
self.type = type
self.total_mem_ratio = total_mem
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields)
else:
self.plotter = None
def firstTime(self, total_mem):
num = len(self.VM_Infos)
share = total_mem / num
for name, info in self.VM_Infos.iteritems():
info.setAttribute('Allocated', share)
info.setBalloon()
def dynamic_allocate(self):
ranks = {}
for name, info in self.VM_Infos.iteritems():
rate = info.getAttribute('Rate')
ranks[info] = rate
rank_l = sorted(ranks.items(), lambda x, y: cmp(x[1], y[1]))
max_info = None
max_share = 0
max_index = 0
min_info = None
min_share = 0
min_index = 0
for i in range(0, len(rank_l)):
info = rank_l[i][0]
if info.dist2high() > 0:
min_info = info
min_share = info.dist2high()
min_index = i
break
for i in range(len(rank_l) - 1, -1, -1):
info = rank_l[i][0]
if info.dist2low() > 0:
max_info = info
max_share = info.dist2low()
max_index = i
break
if min_index >= max_index:
return
share = min(max_share, min_share, Min_share)
max_info.addAttribute('Allocated', -share)
min_info.addAttribute('Allocated', share)
for name, info in self.VM_Infos.iteritems():
info.setBalloon()
def meanByFree(self, active, inactive):
total = 0
for info in inactive:
share = min(info.getAttribute('Free') - Max_free, \
info.getAttribute('Allocated') - info.getAttribute('Min'))
if share <= 0:
continue
info.addAttribute('Allocated', -share)
total += share
ave_share = total / len(active)
for info in active:
info.addAttribute('Allocated', ave_share)
for name, info in self.VM_Infos.iteritems():
info.setBalloon()
def check(self):
active = []
inactive = []
for name, info in self.VM_Infos.iteritems():
if info.getAttribute('Free') < Max_free:
active.append(info)
else:
inactive.append(info)
if len(active) == 0:
return False
elif len(inactive) == 0:
return True
else:
self.meanByFree(active, inactive)
return False
def evaluate(self, host, guest_list):
total_mem = host.mem_available * float(self.total_mem_ratio)
for guest in guest_list:
name = guest.Prop('name')
if name not in self.VM_Infos or self.VM_Infos[name] is None:
info = VM_Info(name)
info.initAttribute(guest)
self.VM_Infos[name] = info
else:
info = self.VM_Infos[name]
info.update(guest)
if self.start:
self.firstTime(total_mem)
self.start = False
return
if self.check():
self.dynamic_allocate()
if self.plotter is not None:
for name, info in self.VM_Infos.iteritems():
data = info.getData(self.fields)
self.plotter.plot(data)
class Old_RPPolicy:
def __init__(self):
self.logger = logging.getLogger('mom.RPPolicy')
self.policy_sem = threading.Semaphore()
self.algorithm = Algorithm()
self.fields = set([
'VM', 'User', 'Balloon', 'Swap_usage', 'Active_Mem', 'Working_Set',
'Allocated', 'Congestion', 'Predict', 'T_workset', 'T_used'
])
self.start = True
def _validate_config(self, host, guest_list):
for id, guest in guest_list:
total = guest.GetVar('mem_available')
if total <= self.vmpres_threshold:
self.logger.error("invalid 'vmpres_threshold'")
return False
total = host.GetVar('mem_available')
if total <= self.hostpres_threshold:
self.logger.error("invalid 'hostpres_threshold'")
return False
return True
def set_policy(self, type, total_mem, plot_dir):
self.type = type
self.total_mem_ratio = total_mem
name = 'Policy'
if plot_dir != '':
self.plotter = Plotter(plot_dir, name)
self.plotter.setFields(self.fields)
else:
self.plotter = None
def setBalloon(self, targets, vms, datas, flag):
for name, vm in vms.iteritems():
target = targets[name]
vm.Control('balloon_target', target)
datas[name]['Allocated'] = target
datas[name]['Congestion'] = flag
def initAllocate(self, vms, total_mem, datas, flag):
share = total_mem / len(vms)
for name, vm in vms.iteritems():
vm.Control('balloon_target', share)
datas[name]['Allocated'] = share
datas[name]['Congestion'] = flag
def evaluate(self, host, guest_list):
datas = {}
vms = {}
weights_user = {}
weights_vms = {}
active_mems = {}
config_mems = {}
swap_usages = {}
worksets = {}
low_limit = {}
config_mems_user = {}
low_limit_user = {}
balloons = {}
all_statistics = {}
total_mem = host.mem_available * float(self.total_mem_ratio)
for guest in guest_list:
name = guest.Prop('name')
weight_user = guest.Prop('weight-user')
weight_vm = guest.Prop('weight-vm')
user = guest.Prop('userID')
self.algorithm.vm_user[name] = user
free_mem = guest.mem_unused
config_mem = guest.balloon_max
balloon_cur = guest.balloon_cur
swap_usage = guest.swap_usage
all_statistics[name] = guest.statistics
vms[name] = guest
weights_vms[name] = weight_vm
weights_user[user] = weight_user
active_mems[name] = balloon_cur - free_mem + 150000
config_mems[name] = config_mem
swap_usages[name] = swap_usage
worksets[name] = swap_usage + active_mems[name]
low_limit[name] = 400000
balloons[name] = balloon_cur
datas[name] = {}
datas[name]['VM'] = name
datas[name]['User'] = user
datas[name]['Balloon'] = config_mem - balloon_cur
datas[name]['Swap_usage'] = swap_usage
datas[name]['Active_Mem'] = active_mems[name]
datas[name]['Working_Set'] = worksets[name]
self.logger.info("basic info: (%s) free_memory: %s, swap size: %s, workset: %s", \
name, free_mem, swap_usages[name], worksets[name])
if user not in config_mems_user or config_mems_user[user] is None:
config_mems_user[user] = config_mem
else:
config_mems_user[user] += config_mem
if user not in low_limit_user or low_limit_user[user] is None:
low_limit_user[user] = low_limit[name]
else:
low_limit_user[user] += low_limit[name]
total_predict, predict_user = self.algorithm.predictor(worksets, datas)
# self.logger.info("predictors is %s", \
# predict_user)
if total_predict > total_mem:
#self.logger.info("Congestion !!!")
# user_allocate = self.algorithm.user_fairness(weights_user, predict_user, low_limit_user, config_mems_user, total_mem)
# self.logger.info("user_allocate is %s", \
# user_allocate)
# vm_allocate = self.algorithm.vm_fairness(user_allocate, weights_vms, low_limit, config_mems)
vm_allocate = self.algorithm.perf_diff(weights_vms, low_limit,
config_mems, all_statistics,
total_mem)
self.logger.info("vm_allocate is %s", \
vm_allocate)
self.setBalloon(vm_allocate, vms, datas, 1)
self.algorithm.update(vm_allocate, worksets, active_mems, datas)
self.algorithm.check(weights_user, weights_vms)
else:
if self.start:
self.initAllocate(vms, total_mem, datas, 0)
self.start = False
elif self.algorithm.check_adjust(balloons):
vm_allocate = self.algorithm.meanByWork(
total_predict, total_mem)
self.setBalloon(vm_allocate, vms, datas, 0)
else:
for name, allocated in balloons.iteritems():
datas[name]['Allocated'] = allocated
datas[name]['Congestion'] = 0
if self.plotter is not None:
for name, data in datas.iteritems():
self.plotter.plot(data)
