def save_all(proc_num):
assert os.path.isdir(conf.SAVE_PATH)
domain_dict = XenStore.domain_name_id_map()
del domain_dict['Domain-0']
names = domain_dict.keys()
pool = multiprocessing.Pool(proc_num)
pool.map(save, names)
def save_all(proc_num):
assert os.path.isdir(conf.SAVE_PATH)
domain_dict = XenStore.domain_name_id_map()
del domain_dict['Domain-0']
names = domain_dict.keys()
pool = multiprocessing.Pool(proc_num)
pool.map(save, names)
def gen_meta(vps_mgr, domain_dict, vps_id):
xv = XenVPS(vps_id)
vps = vps_mgr.query_vps(vps_id)
if not vps_mgr.vps_is_valid(vps):
print "no backend data for %s " % vps_id
return
domain_id = domain_dict[xv.name]
vif_datas = XenStore.get_vif_by_domain_id(domain_id)
vps_mgr.setup_vps(xv, vps)
#xv.add_extra_storage (1, 964) #TEST
vpsops = VPSOps(vps_mgr.logger)
if len(vif_datas.values()) == 1:
vif_data = vif_datas.values()[0]
if vif_data['online'] == '1':
if vif_data.has_key('vifname'):
xv.vifs[vif_data['vifname']].mac = vif_data['mac']
else:
vif_id = vif_datas.items()[0][0]
vif_name = 'vif%s.%s' % (domain_id, vif_id)
del xv.vifs[xv.name]
xv.add_netinf_ext(ip=xv.ip,
netmask=xv.netmask,
mac=vif_data['mac'])
#vpsops.create_xen_config (xv)
vpsops.save_vps_meta(xv, override=True)
def main ():
vps_mgr = VPSMgr ()
domain_dict = XenStore.domain_name_id_map()
for k, v in domain_dict.iteritems ():
om = re.match (r'^vps(\d+)$', k)
if om:
print int(om.group(1))
gen_meta (vps_mgr, domain_dict, int(om.group (1)))
def main():
vps_mgr = VPSMgr()
domain_dict = XenStore.domain_name_id_map()
for k, v in domain_dict.iteritems():
om = re.match(r'^vps(\d+)$', k)
if om:
print int(om.group(1))
gen_meta(vps_mgr, domain_dict, int(om.group(1)))
def vps_open(self, vps_info, vps_image=None, is_new=True):
vps_id = vps_info.id
self.logger.info("to open vps %s" % (vps_id))
if vps_info.host_id != self.host_id:
msg = "vpsopen : vps %s host_id=%s != current host %s , abort" % (
vps_id, vps_info.host_id, self.host_id)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
if not self.vpsinfo_check_ip(vps_info):
msg = "no ip with vps %s" % (vps_id)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
xv = XenVPS(vps_id)
try:
domain_dict = XenStore.domain_name_id_map()
limit = None
if 'VPS_NUM_LIMIT' in dir(conf):
limit = conf.VPS_NUM_LIMIT
if limit and len(domain_dict.keys()) >= limit + 1:
msg = "vps open: cannot open more than %d vps" % (limit)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
self.setup_vps(xv, vps_info)
if xv.is_running():
msg = "vps %s is running" % (vps_id)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, True, msg)
return
if vps_info.state in [VM_STATE.PAY, VM_STATE.OPEN, VM_STATE.CLOSE]:
if self.vpsops.is_normal_exists(vps_id):
xv.check_storage_integrity()
xv.check_xen_config()
if not xv.is_running():
self.logger.info(
"seems vps %s was not closed, try to boot" %
(vps_id))
self.vpsops._boot_and_test(xv, is_new=False)
elif self.vpsops.is_trash_exists(vps_id):
self.vpsops.reopen_vps(vps_id, xv)
else:
self.vpsops.create_vps(xv, vps_image, is_new)
else:
msg = "vps%s state is %s(%s)" % (str(vps_id), vps_info.state,
VM_STATE_CN[vps_info.state])
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
except Exception, e:
self.logger.exception("vps %s: %s" % (str(vps_id), str(e)))
self.done_task(CMD.OPEN, vps_id, False, "error, " + str(e))
return
def vps_open(self, vps_info, vps_image=None, is_new=True):
vps_id = vps_info.id
self.logger.info("to open vps %s" % (vps_id))
if vps_info.host_id != self.host_id:
msg = "vpsopen : vps %s host_id=%s != current host %s , abort" % (
vps_id, vps_info.host_id, self.host_id)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
if not self.vpsinfo_check_ip(vps_info):
msg = "no ip with vps %s" % (vps_id)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
xv = XenVPS(vps_id)
try:
domain_dict = XenStore.domain_name_id_map()
limit = None
if 'VPS_NUM_LIMIT' in dir(conf):
limit = conf.VPS_NUM_LIMIT
if limit and len(domain_dict.keys()) >= limit + 1:
msg = "vps open: cannot open more than %d vps" % (limit)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
self.setup_vps(xv, vps_info)
if xv.is_running():
msg = "vps %s is running" % (vps_id)
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, True, msg)
return
if vps_info.state in [VM_STATE.PAY, VM_STATE.OPEN, VM_STATE.CLOSE]:
if self.vpsops.is_normal_exists(vps_id):
xv.check_storage_integrity()
xv.check_xen_config()
if not xv.is_running():
self.logger.info(
"seems vps %s was not closed, try to boot" % (vps_id))
self.vpsops._boot_and_test(xv, is_new=False)
elif self.vpsops.is_trash_exists(vps_id):
self.vpsops.reopen_vps(vps_id, xv)
else:
self.vpsops.create_vps(xv, vps_image, is_new)
else:
msg = "vps%s state is %s(%s)" % (
str(vps_id), vps_info.state, VM_STATE_CN[vps_info.state])
self.logger.error(msg)
self.done_task(CMD.OPEN, vps_id, False, msg)
return
except Exception, e:
self.logger.exception("vps %s: %s" % (str(vps_id), str(e)))
self.done_task(CMD.OPEN, vps_id, False, "error, " + str(e))
return
def gen_meta (vps_mgr, domain_dict, vps_id):
xv = XenVPS (vps_id)
vps = vps_mgr.query_vps (vps_id)
if not vps_mgr.vps_is_valid (vps):
print "no backend data for %s " % vps_id
return
domain_id = domain_dict[xv.name]
vif_datas = XenStore.get_vif_by_domain_id (domain_id)
vps_mgr.setup_vps (xv, vps)
#xv.add_extra_storage (1, 964) #TEST
vpsops = VPSOps (vps_mgr.logger)
if len (vif_datas.values ()) == 1:
vif_data = vif_datas.values()[0]
if vif_data['online'] == '1':
if vif_data.has_key ('vifname'):
xv.vifs[vif_data['vifname']].mac = vif_data['mac']
else:
vif_id = vif_datas.items ()[0][0]
vif_name = 'vif%s.%s' % (domain_id, vif_id)
del xv.vifs[xv.name]
xv.add_netinf_ext (ip=xv.ip, netmask=xv.netmask, mac=vif_data['mac'])
#vpsops.create_xen_config (xv)
vpsops.save_vps_meta (xv, override=True)
def running_count(self):
domain_dict = XenStore.domain_name_id_map()
del domain_dict['Domain-0']
return len(domain_dict)
#!/usr/bin/env python
import _env
import pprint
from ops.ixen import XenStore
print XenStore.domain_name_id_map ()
#pprint.pprint( XenStore._get_dict ("/local/domain/0/backend/vif/114/0"))
#pprint.pprint( XenStore._get_tree ("/local/domain/0/backend"))
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 :
class VPSMgr(object):
""" all exception should catch and log in this class """
VERSION = 1
def __init__(self):
self.logger = Log("vps_mgr", config=conf)
self.logger_net = Log("vps_mgr_net", config=conf)
self.logger_misc = Log("misc", config=conf)
self.logger_debug = Log("debug", config=conf)
self.host_id = conf.HOST_ID
self.vpsops = VPSOps(self.logger)
self.handlers = {
CMD.OPEN: self.__class__.vps_open,
CMD.REBOOT: self.__class__.vps_reboot,
CMD.CLOSE: self.__class__.vps_close,
CMD.OS: self.__class__.vps_reinstall_os,
CMD.UPGRADE: self.__class__.vps_upgrade,
CMD.BANDWIDTH: self.__class__.vps_set_bandwidth,
CMD.RM: self.__class__.vps_delete,
CMD.PRE_SYNC: self.__class__.vps_hot_sync,
CMD.MIGRATE: self.__class__.vps_migrate,
CMD.RESET_PW: self.__class__.vps_reset_pw,
}
self._locker = threading.Lock()
self._vps_locker = dict()
self.xenstat = XenStat()
self.timer = TimerEvents(time.time, self.logger_misc)
assert conf.MONITOR_COLLECT_INV > 0
self.last_netflow = None
self.last_diskstat = None
self.monitor_inv = conf.MONITOR_COLLECT_INV
self.last_monitor_ts = None
self.timer.add_timer(conf.MONITOR_COLLECT_INV, self.monitor_vps)
self.timer.add_timer(12 * 3600, self.refresh_host_space)
self.workers = []
self.running = False
def _try_lock_vps(self, cmd, vps_id):
self._locker.acquire()
if self._vps_locker.has_key(vps_id):
_cmd = self._vps_locker.get(vps_id)
self.logger_debug.info(
"CMD %s try to lock vps%s failed: locked by CMD %s" %
(CMD._get_name(cmd), vps_id, CMD._get_name(_cmd)))
res = False
else:
self._vps_locker[vps_id] = cmd
res = True
self._locker.release()
return res
def _unlock_vps(self, cmd, vps_id):
self._locker.acquire()
try:
_cmd = self._vps_locker.get(vps_id)
if _cmd == cmd:
del self._vps_locker[vps_id]
except KeyError:
pass
self._locker.release()
def rpc_connect(self):
rpc = SAAS_Client(self.host_id, self.logger_debug)
rpc.connect()
return rpc
def monitor_vps(self):
net_result = None
disk_result = None
try:
net_result = netflow.read_proc()
disk_devs = glob.glob("/dev/main/vps*")
if 'MAIN_DISK' in dir(conf):
disk_devs.append(conf.MAIN_DISK)
disk_result = diskstat.read_stat(disk_devs)
except Exception, e:
self.logger_misc.exception(
"cannot read netflow data from proc: %s" % (str(e)))
return
ts = time.time()
dom_map = XenStore.domain_name_id_map()
dom_names = dom_map.keys()
self.xenstat.run(dom_names)
payload = CarbonPayload()
try:
payload.append("host.cpu.%s.all" % (self.host_id), ts,
self.xenstat.total_cpu)
for dom_name in dom_names:
om = re.match("^vps(\d+)$", dom_name)
if not om:
# dom0
dom_cpu = self.xenstat.dom_dict.get(dom_name)
if dom_cpu:
payload.append("host.cpu.%s.dom0" % (self.host_id),
dom_cpu['ts'], dom_cpu['cpu_avg'])
if 'MAIN_DISK' in dir(conf) and self.last_diskstat:
t_elapse = ts - self.last_monitor_ts
v = disk_result.get(conf.MAIN_DISK)
last_v = self.last_diskstat.get(conf.MAIN_DISK)
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
payload.append("host.io.%d.ops.read" % (self.host_id),
ts, read_ops)
payload.append("host.io.%d.ops.write" % (self.host_id),
ts, write_ops)
payload.append(
"host.io.%s.traffic.read" % (self.host_id), ts,
read_byte)
payload.append(
"host.io.%s.traffic.write" % (self.host_id), ts,
write_byte)
payload.append("host.io.%s.util" % (self.host_id), ts,
util)
print conf.MAIN_DISK, read_ops, write_ops, read_byte, write_byte, util
if self.last_netflow:
t_elapse = ts - self.last_monitor_ts
v = net_result.get(conf.EXT_INF)
last_v = self.last_netflow.get(conf.EXT_INF)
_in = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_out = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_in_pp = (v[2] - last_v[2]) / t_elapse
_out_pp = (v[3] - last_v[3]) / t_elapse
payload.append(
"host.netflow.%d.ext.in" % (self.host_id), ts, _in)
payload.append(
"host.netflow.%d.ext.out" % (self.host_id), ts,
_out)
payload.append(
"host.netflow.%d.ext_pp.in" % (self.host_id), ts,
_in_pp > 0 and _in_pp or 0)
payload.append(
"host.netflow.%d.ext_pp.out" % (self.host_id), ts,
_out_pp > 0 and _out_pp or 0)
v = net_result.get(conf.INT_INF)
last_v = self.last_netflow.get(conf.INT_INF)
_in = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_out = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_in_pp = (v[2] - last_v[2]) / t_elapse
_out_pp = (v[3] - last_v[3]) / t_elapse
payload.append(
"host.netflow.%d.int.in" % (self.host_id), ts, _in)
payload.append(
"host.netflow.%d.int.out" % (self.host_id), ts,
_out)
payload.append(
"host.netflow.%d.int_pp.in" % (self.host_id), ts,
_in_pp > 0 and _in_pp or 0)
payload.append(
"host.netflow.%d.int_pp.out" % (self.host_id), ts,
_out_pp > 0 and _out_pp or 0)
else:
vps_id = int(om.group(1))
xv = self.vpsops.load_vps_meta(vps_id)
dom_cpu = self.xenstat.dom_dict.get(dom_name)
if dom_cpu:
payload.append("vps.cpu.%s" % (vps_id), dom_cpu['ts'],
dom_cpu['cpu_avg'])
if not self.last_netflow or not self.last_diskstat:
break
# net
ifname = dom_name
vif = xv.vifs.get(ifname)
v = net_result.get(ifname)
last_v = self.last_netflow.get(ifname)
t_elapse = ts - self.last_monitor_ts
if v and last_v:
# direction of vps bridged network interface needs to
# be reversed
_in = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_out = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_in = (vif.bandwidth and vif.bandwidth * 1024 * 1024 <
_in) and vif.bandwidth * 1024 * 1024 or _in
_out = (vif.bandwidth and vif.bandwidth * 1024 * 1024 <
_out) and vif.bandwidth * 1024 * 1024 or _out
payload.append("vps.netflow.%d.in" % (vps_id), ts, _in)
payload.append("vps.netflow.%d.out" % (vps_id), ts,
_out)
if conf.LARGE_NETFLOW and _in >= conf.LARGE_NETFLOW or _out >= conf.LARGE_NETFLOW:
self.logger_misc.warn(
"%s in: %.3f mbps, out: %.3f mbps" %
(ifname, _in / 1024.0 / 1024.0,
_out / 1024.0 / 1024.0))
# disk
if conf.USE_LVM and self.last_diskstat:
for disk in xv.data_disks.values():
v = disk_result.get(disk.dev)
last_v = self.last_diskstat.get(disk.dev)
if not last_v:
continue
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
print disk.xen_dev
payload.append(
"vps.io.%d.%s.ops.read" %
(vps_id, disk.xen_dev), ts, read_ops)
payload.append(
"vps.io.%d.%s.ops.write" %
(vps_id, disk.xen_dev), ts, write_ops)
payload.append(
"vps.io.%d.%s.traffic.read" %
(vps_id, disk.xen_dev), ts, read_byte)
payload.append(
"vps.io.%d.%s.traffic.write" %
(vps_id, disk.xen_dev), ts, write_byte)
payload.append(
"vps.io.%d.%s.util" % (vps_id, disk.xen_dev),
ts, util)
v = disk_result.get(xv.swap_store.dev)
last_v = self.last_diskstat.get(xv.swap_store.dev)
if v and last_v:
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
payload.append(
"vps.io.%d.swap.ops.read" % (vps_id), ts,
read_ops)
payload.append(
"vps.io.%d.swap.ops.write" % (vps_id), ts,
write_ops)
payload.append(
"vps.io.%d.swap.traffic.read" % (vps_id), ts,
read_byte)
payload.append(
"vps.io.%d.swap.traffic.write" % (vps_id), ts,
write_byte)
payload.append("vps.io.%d.swap.util" % (vps_id),
ts, util)
self.last_netflow = net_result
self.last_diskstat = disk_result
self.last_monitor_ts = ts
except Exception, e:
self.logger_misc.exception(e)
return
def running_count(self):
domain_dict = XenStore.domain_name_id_map()
del domain_dict['Domain-0']
return len(domain_dict)
cpu_diff = float(info['cpu_time'] - last_info['cpu_time'])
ts_diff = float(info['ts'] - last_info['ts'])
info['cpu_avg'] = cpu_diff / ts_diff / int(info['vcpus']) * 100
if info['cpu_avg'] > 100:
info['cpu_avg'] = 100
total_vcpu += int(info['vcpus'])
total_cpu_time_diff += cpu_diff
total_ts_diff += ts_diff
else:
info['cpu_avg'] = 0
new_dom_dict[dom_name] = info
if total_ts_diff:
ts_avg = total_ts_diff / len(dom_names)
self.total_cpu = total_cpu_time_diff / ts_avg * 100
self.dom_dict = new_dom_dict
if __name__ == '__main__':
from ops.ixen import get_xen_inf, XenStore
xs = XenStat()
dom_map = XenStore.domain_name_id_map()
dom_names = dom_map.keys()
while True:
xs.run(dom_names)
time.sleep(1)
dom0 = xs.dom_dict.get('Domain-0')
print dom0['cpu_avg']
print xs.total_cpu
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 :
ts_diff = float(info['ts'] - last_info['ts'])
info['cpu_avg'] = cpu_diff / ts_diff / int(info['vcpus']) * 100
if info['cpu_avg'] > 100:
info['cpu_avg'] = 100
total_vcpu += int(info['vcpus'])
total_cpu_time_diff += cpu_diff
total_ts_diff += ts_diff
else:
info['cpu_avg'] = 0
new_dom_dict[dom_name] = info
if total_ts_diff:
ts_avg = total_ts_diff / len(dom_names)
self.total_cpu = total_cpu_time_diff / ts_avg * 100
self.dom_dict = new_dom_dict
if __name__ == '__main__':
from ops.ixen import get_xen_inf, XenStore
xs = XenStat()
dom_map = XenStore.domain_name_id_map()
dom_names = dom_map.keys()
while True:
xs.run(dom_names)
time.sleep(1)
dom0 = xs.dom_dict.get('Domain-0')
print dom0['cpu_avg']
print xs.total_cpu
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 :
#!/usr/bin/env python
import _env
import pprint
from ops.ixen import XenStore
print XenStore.domain_name_id_map()
#pprint.pprint( XenStore._get_dict ("/local/domain/0/backend/vif/114/0"))
#pprint.pprint( XenStore._get_tree ("/local/domain/0/backend"))
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 :
