def wait_unit_el_present(self, keys):
"""
Display wait,wait until element display, time out 15 sec
"""
wait = WebDriverWait(self.driver, 15)
key = keys[0]
value = keys[1]
if key in ['XPATH', 'xpath']:
locator = (By.XPATH, value)
elif key in ['link_text', 'LINK_TEXT']:
locator = (By.LINK_TEXT, value)
elif key in ['ID','id']:
locator = (By.ID, value)
elif key in ['NAME', 'name']:
locator = (By.NAME, value)
elif key in ['CLASS_NAME', 'class_name']:
element = (By.CLASS_NAME, value)
elif key in ['TAG', 'tag']:
element = (By.TAG_NAME, value)
try:
element = wait.until(ec.presence_of_element_located(locator))
except Exception:
Log.exception("Failed to wait element!")
self.get_windows_img()
self.sleep(2)
return element
def test_create_vps0 (self):
print "create vps00"
logger = Log ("test", config=conf)
vpsops = VPSOps (logger)
vps = XenVPS (0)
try:
vps.setup (os_id=20001, vcpu=1, mem_m=512, disk_g=7, ip="113.11.199.3", netmask="255.255.255.0", gateway="113.11.199.1", root_pw="fdfdfd")
#vps.add_extra_storage (disk_id=1, size_g=1, fs_type='ext3')
#vps.add_extra_storage (disk_id=2, size_g=0.5, fs_type='ext4')
#vps.setup (os_id=10001, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
#vps.setup (os_id=10002, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
#vps.setup (os_id=30001, vcpu=1, mem_m=512, disk_g=7, ip="10.10.2.2", netmask="255.255.255.0", gateway="10.10.2.1", root_pw="root")
#vps.setup (os_id=1, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
#vps.setup (os_id=10000, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
#vps.setup (os_id=20001, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
#vps.setup (os_id=10003, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
#vps.setup (os_id=20001, vcpu=1, mem_m=512, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
print vps.gen_xenpv_config ()
#vpsops.create_vps (vps, vps_image='/data/vps/images/arch-2011.08.19-i386-fs-ext3.tar.gz')
vpsops.create_vps (vps)
except Exception, e:
print str(e)
logger.exception (e)
raise e
def hotsync_partition(lv_dev, dest_ip, speed=None):
logger = Log("vps_mgr", config=conf)
try:
client = MigrateClient(logger, dest_ip)
client.snapshot_sync(lv_dev, speed=speed)
print "ok"
except Exception, e:
logger.exception(e)
raise e
def sync_partition(dev, dest_ip, speed=None):
logger = Log("vps_mgr", config=conf)
try:
client = MigrateClient(logger, dest_ip)
client.sync_partition(dev, speed=speed)
print "ok"
except Exception, e:
logger.exception(e)
raise e
def delete_customer(self, customer_name):
"""
Delete a customer
"""
try:
self.check_customer(customer_name)
self.click(self.delete_btn_ele)
self.click(self.ok_btn_ele)
except Exception:
Log.exception("No customer is found!")
def test_mem_too_big (self):
print "test mem too big"
vps = XenVPS (0)
logger = Log ("test", config=conf)
vps.setup (os_id=50001, vcpu=1, mem_m=500000, disk_g=7, ip="10.10.1.2", netmask="255.255.255.0", gateway="10.10.1.1", root_pw="fdfdfd")
try:
vps.check_resource_avail ()
except Exception, e:
logger.exception (e)
print "exception caught", type (e), str(e)
return
def test_vps0 (self):
print "create vps00"
logger = Log ("test", config=conf)
vpsops = VPSOps (logger)
xv = XenVPS (0)
try:
xv.setup (os_id=10001, vcpu=1, mem_m=512, disk_g=7, ip="10.10.2.2", netmask="255.255.255.0", gateway="10.10.2.1", root_pw="fdfdfd")
xv.add_extra_storage (disk_id=1, size_g=1, fs_type='ext3')
print xv.gen_xenpv_config ()
vpsops.create_vps (xv)
except Exception, e:
logger.exception (e)
raise e
class VPSMgr(object):
""" all exception should catch and log in this class """
VERSION = 1
def __init__(self):
self.logger = Log("vps_mgr", config=conf)
self.logger_err = Log("vps_mgr_err", config=conf)
self.logger_misc = Log("misc", config=conf)
self.host_id = conf.HOST_ID
self.handlers = {
Cmd.OPEN: self.__class__.vps_open,
Cmd.REBOOT: self.__class__.vps_reboot,
Cmd.CLOSE: self.__class__.vps_close,
Cmd.OS: self.__class__.vps_reinstall_os,
}
self.timer = TimerEvents(time.time, self.logger_misc)
assert conf.NETFLOW_COLLECT_INV > 0
self.timer.add_timer(conf.NETFLOW_COLLECT_INV, self.send_netflow)
self.timer.add_timer(12 * 3600, self.refresh_host_space)
self.workers = []
self.running = False
def get_client(self):
return get_client(VPS)
def send_netflow(self):
result = None
try:
result = netflow.read_proc()
except Exception, e:
self.logger_misc.exception(
"cannot read netflow data from proc: %s" % (str(e)))
return
ts = time.time()
netflow_list = list()
try:
for ifname, v in result.iteritems():
om = re.match("^vps(\d+)$", ifname)
if not om:
continue
vps_id = int(om.group(1))
if vps_id <= 0:
continue
# direction of vps bridged network interface needs to be reversed
netflow_list.append(NetFlow(vps_id, rx=v[1], tx=v[0]))
except Exception, e:
self.logger_misc.exception("netflow data format error: %s" %
(str(e)))
return
def main():
logger = Log("vps_mgr", config=conf)
mgr = VPSMgr()
host_list = None
try:
rpc = mgr.rpc_connect()
try:
host_list = rpc.host_list()
finally:
rpc.close()
update_iplist(host_list)
except Exception, e:
print e
logger.exception(e)
return
def main():
logger = Log("vps_mgr", config=conf)
mgr = VPSMgr()
host_list = None
try:
rpc = mgr.rpc_connect()
try:
host_list = rpc.host_list()
finally:
rpc.close()
update_iplist(host_list)
except Exception, e:
print e
logger.exception(e)
return
class VPSMgr (object):
""" all exception should catch and log in this class """
VERSION = 1
def __init__ (self):
self.logger = Log ("vps_mgr", config=conf)
self.logger_err = Log ("vps_mgr_err", config=conf)
self.logger_misc = Log ("misc", config=conf)
self.host_id = conf.HOST_ID
self.handlers = {
Cmd.OPEN: self.__class__.vps_open,
Cmd.REBOOT: self.__class__.vps_reboot,
Cmd.CLOSE: self.__class__.vps_close,
Cmd.OS: self.__class__.vps_reinstall_os,
}
self.timer = TimerEvents (time.time, self.logger_misc)
assert conf.NETFLOW_COLLECT_INV > 0
self.timer.add_timer (conf.NETFLOW_COLLECT_INV, self.send_netflow)
self.timer.add_timer (12 * 3600, self.refresh_host_space)
self.workers = []
self.running = False
def get_client (self):
return get_client (VPS)
def send_netflow (self):
result = None
try:
result = netflow.read_proc ()
except Exception, e:
self.logger_misc.exception ("cannot read netflow data from proc: %s" % (str(e)))
return
ts = time.time ()
netflow_list = list ()
try:
for ifname, v in result.iteritems ():
om = re.match ("^vps(\d+)$", ifname)
if not om:
continue
vps_id = int(om.group (1))
if vps_id <= 0:
continue
# direction of vps bridged network interface needs to be reversed
netflow_list.append (NetFlow (vps_id, rx=v[1], tx=v[0]))
except Exception, e:
self.logger_misc.exception ("netflow data format error: %s" % (str(e)))
return
class RaidMonitor(object):
def __init__(self):
self.logger = Log("raid_mon", config=conf)
self.is_running = False
self.last_state = True
self.vps_mgr = VPSMgr()
self.hostname = socket.gethostname()
def start(self):
if self.is_running:
return
self.is_running = True
self.logger.info("started")
def stop(self):
if not self.is_running:
return
self.is_running = False
def send_alarm(self, msg):
rpc = self.vps_mgr.rpc_connect()
try:
rpc.alarm("%s: raid_mon: %s" % (self.hostname, msg))
finally:
rpc.close()
def check(self):
cmd = """MegaCli64 -pdlist -aall | grep -i 'firmware state:' | grep -P -v -i "online|Unconfigured\(good\)" """
try:
out, err = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
msg = out + err
if msg:
self.logger.error(msg)
if self.last_state:
self.last_state = False
self.send_alarm("error, %s" % (out))
self.logger.error("alarm sent")
else:
self.logger.info("ok")
if not self.last_state:
self.send_alarm("ok")
self.last_state = True
except Exception, e:
self.logger.exception(e)
class RaidMonitor(object):
def __init__(self):
self.logger = Log("raid_mon", config=conf)
self.is_running = False
self.last_state = True
self.vps_mgr = VPSMgr()
self.hostname = socket.gethostname()
def start(self):
if self.is_running:
return
self.is_running = True
self.logger.info("started")
def stop(self):
if not self.is_running:
return
self.is_running = False
def send_alarm(self, msg):
rpc = self.vps_mgr.rpc_connect()
try:
rpc.alarm("%s: raid_mon: %s" % (self.hostname, msg))
finally:
rpc.close()
def check(self):
cmd = """MegaCli64 -pdlist -aall | grep -i 'firmware state:' | grep -P -v -i "online|Unconfigured\(good\)" """
try:
out, err = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
msg = out + err
if msg:
self.logger.error(msg)
if self.last_state:
self.last_state = False
self.send_alarm("error, %s" % (out))
self.logger.error("alarm sent")
else:
self.logger.info("ok")
if not self.last_state:
self.send_alarm("ok")
self.last_state = True
except Exception, e:
self.logger.exception(e)
class TransWarpServer (TransWarpBase):
def __init__(self):
TransWarpBase.__init__(self)
self.logger = Log("server", config=config)
self.engine.set_logger(self.logger)
self.addr = config.SERVER_ADDR
self.auth_keys = config.ACCEPT_KEYS
self.passive_sock = None
def _new_client(self, sock):
conn = Connection(sock)
print "new %s" % (str(conn.peer))
self.engine.read_unblock(
conn, self.head_len, self._on_recv_head, None, cb_args=(self._auth, ))
def _auth(self, cli_conn):
auth_data = None
try:
auth_data = proto.AuthData.deserialize(
cli_conn.get_readbuf(), self.auth_keys)
except Exception, e:
self.logger.exception("peer %s %s" % (cli_conn.peer, str(e)))
if not auth_data:
self.logger.warn("peer %s not authorized" % (str(cli_conn.peer)))
self.engine.close_conn(cli_conn)
return
client = proto.ClientData(
auth_data.r_host, auth_data.r_port, cli_conn,
auth_data.seed, auth_data.key)
self.engine.remove_conn(cli_conn)
self.client_conn[client.client_id] = client
self.logger.info("client %s auth" % (client.client_id))
client.cli_state = proto.ClientState.CONNECTING
self.engine.connect_unblock(
(client.r_host, client.r_port), self._on_remote_conn, self._on_remote_conn_err, cb_args=(client, ))
resp = proto.ServerResponse(0, "")
buf = client.crypter_w.encrypt(resp.serialize())
def _write_ok(cli_conn, *args):
client.cli_state = proto.ClientState.CONNECTED
self._check_client_state(client)
return
self.engine.write_unblock(client.cli_conn, proto.pack_head(
len(buf)) + buf, _write_ok, self._on_err, cb_args=(client, ))
def test_mem_too_big(self):
print "test mem too big"
vps = XenVPS(0)
logger = Log("test", config=conf)
vps.setup(os_id=50001,
vcpu=1,
mem_m=500000,
disk_g=7,
ip="10.10.1.2",
netmask="255.255.255.0",
gateway="10.10.1.1",
root_pw="fdfdfd")
try:
vps.check_resource_avail()
except Exception, e:
logger.exception(e)
print "exception caught", type(e), str(e)
return
def main():
bridge = args[0]
vif_name = args[1]
logger = Log("vps_mgr", config=conf)
try:
ovsops = OVSOps()
ofport = ovsops.find_ofport_by_name(vif_name)
if ofport < 0:
logger.error("vif %s ofport=%s, skip it" % (vif_name, ofport))
else:
ovsops.unset_mac_filter(bridge, ofport)
# it's strange that if you unset traffic first, might find ofport==-1
ovsops.unset_traffic_limit(vif_name)
logger.debug("unset %s" % vif_name)
return 0
except Exception, e:
logger.exception(e)
print >> sys.stderr, str(e)
return 0
def main():
bridge = args[0]
vif_name = args[1]
logger = Log("vps_mgr", config=conf)
try:
ovsops = OVSOps()
ofport = ovsops.find_ofport_by_name(vif_name)
if ofport < 0:
logger.error("vif %s ofport=%s, skip it" % (vif_name, ofport))
else:
ovsops.unset_mac_filter(bridge, ofport)
# it's strange that if you unset traffic first, might find ofport==-1
ovsops.unset_traffic_limit(vif_name)
logger.debug("unset %s" % vif_name)
return 0
except Exception, e:
logger.exception(e)
print >> sys.stderr, str(e)
return 0
def test_vps0(self):
print "create vps00"
logger = Log("test", config=conf)
vpsops = VPSOps(logger)
xv = XenVPS(0)
try:
xv.setup(os_id=10001,
vcpu=1,
mem_m=512,
disk_g=7,
ip="10.10.2.2",
netmask="255.255.255.0",
gateway="10.10.2.1",
root_pw="fdfdfd")
xv.add_extra_storage(disk_id=1, size_g=1, fs_type='ext3')
print xv.gen_xenpv_config()
vpsops.create_vps(xv)
except Exception, e:
logger.exception(e)
raise e
def main():
bridge = args[0]
vif_name = args[1]
logger = Log("vps_mgr", config=conf)
vpsops = VPSOps(logger)
logger.debug("set %s" % vif_name)
try:
ovsops = OVSOps()
om = re.match(r'^\w+?(\d+)\w*?$', vif_name)
if not om:
print >> sys.stderr, "wrong vif format %s" % (vif_name)
return 1
vps_id = int(om.group(1))
xv = vpsops.load_vps_meta(vps_id)
vif = xv.vifs.get(vif_name)
if not vif:
logger.error("no vif %s in metadata of %s" % (vif_name, vps_id))
return 1
ofport = ovsops.find_ofport_by_name(vif_name)
if ofport < 0:
logger.error(
"vif %s ofport=%s, fix it by delete the port from bridge " %
(vif_name, ofport))
ovsops.del_port_from_bridge(bridge, vif_name)
ovsops.add_port_to_bridge(bridge, vif_name)
ofport = ovsops.find_ofport_by_name(vif_name)
if ofport < 0:
logger.error("vif %s ofport=%s, impossible " %
(vif_name, ofport))
if ofport >= 0:
ovsops.set_mac_filter(bridge, ofport, vif.ip_dict.keys())
ovsops.unset_traffic_limit(vif_name)
bandwidth = float(vif.bandwidth or 0)
ovsops.set_traffic_limit(vif_name, int(bandwidth * 1000))
print "set vif %s bandwidth %sm/s" % (vif_name, vif.bandwidth)
return 0
except Exception, e:
logger.exception(e)
print >> sys.stderr, str(e)
return 1
class SaasMonitor(object):
def __init__(self):
self.is_running = False
self.hostname = socket.gethostname()
self.logger = Log("saas_mon", config=conf)
self.recover_thres = conf.SAAS_RECOVER_THRESHOLD or(30 * 5)
self.bad_thres = conf.SAAS_BAD_THRESHOLD or 5
self.alarm_q = JobQueue(self.logger)
self.emailalarm = EmailAlarm(self.logger)
self.last_state = True
def start(self):
if self.is_running:
return
self.is_running = True
self.alarm_q.start_worker(1)
self.logger.info("started")
def stop(self):
if not self.is_running:
return
self.is_running = False
self.alarm_q.stop()
def check(self):
vps = None
try:
rpc = SAAS_Client(conf.HOST_ID, self.logger)
rpc.connect()
try:
_id = rpc.todo(CMD.MONITOR)
finally:
rpc.close()
self.logger.info("ok")
return True
except Exception, e:
self.logger.exception(e)
return False
class SaasMonitor(object):
def __init__(self):
self.is_running = False
self.hostname = socket.gethostname()
self.logger = Log("saas_mon", config=conf)
self.recover_thres = conf.SAAS_RECOVER_THRESHOLD or (30 * 5)
self.bad_thres = conf.SAAS_BAD_THRESHOLD or 5
self.alarm_q = JobQueue(self.logger)
self.emailalarm = EmailAlarm(self.logger)
self.last_state = True
def start(self):
if self.is_running:
return
self.is_running = True
self.alarm_q.start_worker(1)
self.logger.info("started")
def stop(self):
if not self.is_running:
return
self.is_running = False
self.alarm_q.stop()
def check(self):
vps = None
try:
rpc = SAAS_Client(conf.HOST_ID, self.logger)
rpc.connect()
try:
_id = rpc.todo(CMD.MONITOR)
finally:
rpc.close()
self.logger.info("ok")
return True
except Exception, e:
self.logger.exception(e)
return False
def main():
bridge = args[0]
vif_name = args[1]
logger = Log("vps_mgr", config=conf)
vpsops = VPSOps(logger)
logger.debug("set %s" % vif_name)
try:
ovsops = OVSOps()
om = re.match(r"^\w+?(\d+)\w*?$", vif_name)
if not om:
print >> sys.stderr, "wrong vif format %s" % (vif_name)
return 1
vps_id = int(om.group(1))
xv = vpsops.load_vps_meta(vps_id)
vif = xv.vifs.get(vif_name)
if not vif:
logger.error("no vif %s in metadata of %s" % (vif_name, vps_id))
return 1
ofport = ovsops.find_ofport_by_name(vif_name)
if ofport < 0:
logger.error("vif %s ofport=%s, fix it by delete the port from bridge " % (vif_name, ofport))
ovsops.del_port_from_bridge(bridge, vif_name)
ovsops.add_port_to_bridge(bridge, vif_name)
ofport = ovsops.find_ofport_by_name(vif_name)
if ofport < 0:
logger.error("vif %s ofport=%s, impossible " % (vif_name, ofport))
if ofport >= 0:
ovsops.set_mac_filter(bridge, ofport, vif.ip_dict.keys())
ovsops.unset_traffic_limit(vif_name)
bandwidth = float(vif.bandwidth or 0)
ovsops.set_traffic_limit(vif_name, int(bandwidth * 1000))
print "set vif %s bandwidth %sm/s" % (vif_name, vif.bandwidth)
return 0
except Exception, e:
logger.exception(e)
print >> sys.stderr, str(e)
return 1
class VPSMgr(object):
""" all exception should catch and log in this class """
VERSION = 1
def __init__(self):
self.logger = Log("vps_mgr", config=conf)
self.logger_net = Log("vps_mgr_net", config=conf)
self.logger_misc = Log("misc", config=conf)
self.logger_debug = Log("debug", config=conf)
self.host_id = conf.HOST_ID
self.vpsops = VPSOps(self.logger)
self.handlers = {
CMD.OPEN: self.__class__.vps_open,
CMD.REBOOT: self.__class__.vps_reboot,
CMD.CLOSE: self.__class__.vps_close,
CMD.OS: self.__class__.vps_reinstall_os,
CMD.UPGRADE: self.__class__.vps_upgrade,
CMD.BANDWIDTH: self.__class__.vps_set_bandwidth,
CMD.RM: self.__class__.vps_delete,
CMD.PRE_SYNC: self.__class__.vps_hot_sync,
CMD.MIGRATE: self.__class__.vps_migrate,
CMD.RESET_PW: self.__class__.vps_reset_pw,
}
self._locker = threading.Lock()
self._vps_locker = dict()
self.xenstat = XenStat()
self.timer = TimerEvents(time.time, self.logger_misc)
assert conf.MONITOR_COLLECT_INV > 0
self.last_netflow = None
self.last_diskstat = None
self.monitor_inv = conf.MONITOR_COLLECT_INV
self.last_monitor_ts = None
self.timer.add_timer(conf.MONITOR_COLLECT_INV, self.monitor_vps)
self.timer.add_timer(12 * 3600, self.refresh_host_space)
self.workers = []
self.running = False
def _try_lock_vps(self, cmd, vps_id):
self._locker.acquire()
if self._vps_locker.has_key(vps_id):
_cmd = self._vps_locker.get(vps_id)
self.logger_debug.info("CMD %s try to lock vps%s failed: locked by CMD %s" % (
CMD._get_name(cmd), vps_id, CMD._get_name(_cmd)
))
res = False
else:
self._vps_locker[vps_id] = cmd
res = True
self._locker.release()
return res
def _unlock_vps(self, cmd, vps_id):
self._locker.acquire()
try:
_cmd = self._vps_locker.get(vps_id)
if _cmd == cmd:
del self._vps_locker[vps_id]
except KeyError:
pass
self._locker.release()
def rpc_connect(self):
rpc = SAAS_Client(self.host_id, self.logger_debug)
rpc.connect()
return rpc
def monitor_vps(self):
net_result = None
disk_result = None
try:
net_result = netflow.read_proc()
disk_devs = glob.glob("/dev/main/vps*")
if 'MAIN_DISK' in dir(conf):
disk_devs.append(conf.MAIN_DISK)
disk_result = diskstat.read_stat(disk_devs)
except Exception, e:
self.logger_misc.exception(
"cannot read netflow data from proc: %s" % (str(e)))
return
ts = time.time()
dom_map = XenStore.domain_name_id_map()
dom_names = dom_map.keys()
self.xenstat.run(dom_names)
payload = CarbonPayload()
try:
payload.append("host.cpu.%s.all" %
(self.host_id), ts, self.xenstat.total_cpu)
for dom_name in dom_names:
om = re.match("^vps(\d+)$", dom_name)
if not om:
# dom0
dom_cpu = self.xenstat.dom_dict.get(dom_name)
if dom_cpu:
payload.append("host.cpu.%s.dom0" %
(self.host_id), dom_cpu['ts'], dom_cpu['cpu_avg'])
if 'MAIN_DISK' in dir(conf) and self.last_diskstat:
t_elapse = ts - self.last_monitor_ts
v = disk_result.get(conf.MAIN_DISK)
last_v = self.last_diskstat.get(conf.MAIN_DISK)
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
payload.append("host.io.%d.ops.read" %
(self.host_id), ts, read_ops)
payload.append("host.io.%d.ops.write" %
(self.host_id), ts, write_ops)
payload.append("host.io.%s.traffic.read" %
(self.host_id), ts, read_byte)
payload.append("host.io.%s.traffic.write" %
(self.host_id), ts, write_byte)
payload.append("host.io.%s.util" %
(self.host_id), ts, util)
print conf.MAIN_DISK, read_ops, write_ops, read_byte, write_byte, util
if self.last_netflow:
t_elapse = ts - self.last_monitor_ts
v = net_result.get(conf.EXT_INF)
last_v = self.last_netflow.get(conf.EXT_INF)
_in = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_out = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_in_pp = (v[2] - last_v[2]) / t_elapse
_out_pp = (v[3] - last_v[3]) / t_elapse
payload.append("host.netflow.%d.ext.in" %
(self.host_id), ts, _in)
payload.append("host.netflow.%d.ext.out" %
(self.host_id), ts, _out)
payload.append("host.netflow.%d.ext_pp.in" %
(self.host_id), ts, _in_pp > 0 and _in_pp or 0)
payload.append("host.netflow.%d.ext_pp.out" %
(self.host_id), ts, _out_pp > 0 and _out_pp or 0)
v = net_result.get(conf.INT_INF)
last_v = self.last_netflow.get(conf.INT_INF)
_in = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_out = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_in_pp = (v[2] - last_v[2]) / t_elapse
_out_pp = (v[3] - last_v[3]) / t_elapse
payload.append("host.netflow.%d.int.in" %
(self.host_id), ts, _in)
payload.append("host.netflow.%d.int.out" %
(self.host_id), ts, _out)
payload.append("host.netflow.%d.int_pp.in" %
(self.host_id), ts, _in_pp > 0 and _in_pp or 0)
payload.append("host.netflow.%d.int_pp.out" %
(self.host_id), ts, _out_pp > 0 and _out_pp or 0)
else:
vps_id = int(om.group(1))
xv = self.vpsops.load_vps_meta(vps_id)
dom_cpu = self.xenstat.dom_dict.get(dom_name)
if dom_cpu:
payload.append("vps.cpu.%s" %
(vps_id), dom_cpu['ts'], dom_cpu['cpu_avg'])
if not self.last_netflow or not self.last_diskstat:
break
# net
ifname = dom_name
vif = xv.vifs.get(ifname)
v = net_result.get(ifname)
last_v = self.last_netflow.get(ifname)
t_elapse = ts - self.last_monitor_ts
if v and last_v:
# direction of vps bridged network interface needs to
# be reversed
_in = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_out = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_in = (vif.bandwidth and vif.bandwidth * 1024 *
1024 < _in) and vif.bandwidth * 1024 * 1024 or _in
_out = (vif.bandwidth and vif.bandwidth * 1024 *
1024 < _out) and vif.bandwidth * 1024 * 1024 or _out
payload.append("vps.netflow.%d.in" %
(vps_id), ts, _in)
payload.append("vps.netflow.%d.out" %
(vps_id), ts, _out)
if conf.LARGE_NETFLOW and _in >= conf.LARGE_NETFLOW or _out >= conf.LARGE_NETFLOW:
self.logger_misc.warn(
"%s in: %.3f mbps, out: %.3f mbps" %
(ifname, _in / 1024.0 / 1024.0, _out / 1024.0 / 1024.0))
# disk
if conf.USE_LVM and self.last_diskstat:
for disk in xv.data_disks.values():
v = disk_result.get(disk.dev)
last_v = self.last_diskstat.get(disk.dev)
if not last_v:
continue
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
print disk.xen_dev
payload.append("vps.io.%d.%s.ops.read" %
(vps_id, disk.xen_dev), ts, read_ops)
payload.append("vps.io.%d.%s.ops.write" %
(vps_id, disk.xen_dev), ts, write_ops)
payload.append("vps.io.%d.%s.traffic.read" %
(vps_id, disk.xen_dev), ts, read_byte)
payload.append("vps.io.%d.%s.traffic.write" %
(vps_id, disk.xen_dev), ts, write_byte)
payload.append("vps.io.%d.%s.util" %
(vps_id, disk.xen_dev), ts, util)
v = disk_result.get(xv.swap_store.dev)
last_v = self.last_diskstat.get(xv.swap_store.dev)
if v and last_v:
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
payload.append("vps.io.%d.swap.ops.read" %
(vps_id), ts, read_ops)
payload.append("vps.io.%d.swap.ops.write" %
(vps_id), ts, write_ops)
payload.append("vps.io.%d.swap.traffic.read" %
(vps_id), ts, read_byte)
payload.append("vps.io.%d.swap.traffic.write" %
(vps_id), ts, write_byte)
payload.append("vps.io.%d.swap.util" %
(vps_id), ts, util)
self.last_netflow = net_result
self.last_diskstat = disk_result
self.last_monitor_ts = ts
except Exception, e:
self.logger_misc.exception(e)
return
class TransWarpClient (TransWarpBase):
def __init__(self):
TransWarpBase.__init__(self)
self.logger = Log("client", config=config)
self.engine.set_logger(self.logger)
self.sock5_addr = config.SOCK5_ADDR
self._sock5_users = 'SOCK5_USERS' in dir(
config) and config.SOCK5_USERS or {}
ip = self.sock5_addr[0]
arr = map(lambda x: chr(int(x)), ip.split("."))
self._sock5_server_id = struct.pack(
"!4cH", arr[0], arr[1], arr[2], arr[3], self.sock5_addr[1])
self.server_addr = config.SERVER_ADDR
self.sock5_sock = None
self.key = config.KEY
def start(self):
if self.is_running:
return
self.sock5_sock = self.engine.listen_addr(
self.sock5_addr, readable_cb=None, new_conn_cb=self._sock5_handshake, backlog=50)
self.is_running = True
def stop(self):
if not self.is_running:
return
self.engine.unlisten(self.sock5_sock)
self.is_running = False
def _send_sock5_unsupport(self, conn):
self.logger.error("peer %s not supported" % (str(conn.peer)))
buf = "%s%s\x00\x01%s" % (VER, "\x07", self._sock5_server_id)
def __write_ok(conn):
self.engine.close_conn(conn)
return
self.engine.write_unblock(conn, buf, __write_ok)
def _send_sock5_reply(self, client, err_no):
if err_no == 0:
status = "\x00"
elif err_no == errno.ENETUNREACH:
status = "\x03"
elif err_no == errno.EHOSTUNREACH:
status = "\x04"
elif err_no == errno.ECONNREFUSED:
status = "\x05"
else:
status = "\x01" # general error
buf = "%s%s\x00\x01%s" % (VER, status, self._sock5_server_id)
def __write_ok(cli_conn, *args):
if err_no == 0:
self.logger.info(
"client %s: sent sock5 response" % (client.client_id))
client.cli_state = proto.ClientState.CONNECTED
self._check_client_state(client)
else:
self.logger.info(
"client %s: sent sock5 err response and close" % (client.client_id))
self.close_client(client)
return
self.engine.write_unblock(
client.cli_conn, buf, __write_ok, self._on_err, cb_args=(client,))
def _on_client_readable(self, cli_conn, client):
# print "client %s client readable" % (client.client_id)
self.stream_to_fix(cli_conn, client.r_conn, client)
def _on_remote_readable(self, r_conn, client):
# print "client %s remote readable" % (client.client_id)
self.fix_to_stream(r_conn, client.cli_conn, client)
def _on_server_connected(self, sock, client):
self.logger.info("client %s connected to server" % (client.client_id))
r_conn = Connection(sock)
_hash = proto.myhash(client.seed, self.key)
auth_data = proto.AuthData(
client.seed, _hash, self.key, client.r_host, client.r_port)
buf = auth_data.serialize()
buf = proto.pack_head(len(buf)) + buf
client.r_conn = r_conn
def __on_remote_respond(r_conn, *args):
resp = None
try:
buf = client.crypter_r.decrypt(r_conn.get_readbuf())
resp = proto.ServerResponse.deserialize(buf)
if resp.err_no:
self.logger.error("client %s: %s %s" %
(client.client_id, resp.err_no, resp.message))
self.close_client(client)
else:
self.logger.info(
"client %s server response" % (client.client_id))
client.r_state = proto.ClientState.CONNECTED
self._check_client_state(client)
except Exception, e:
self.logger.exception(
"client %s: server response error %s" % (client.client_id, str(e)))
self.close_client(client)
return
def __on_read_head(r_conn, *args):
data_len = 0
try:
data_len = proto.unpack_head(r_conn.get_readbuf())
except Exception, e:
self.logger.error(
"client %s remote head invalid" % (client.client_id))
self.close_client(client)
return
if data_len > 0:
self.engine.read_unblock(
r_conn, data_len, __on_remote_respond, self._on_err, cb_args=(client, ))
return
self.logger.error("zero len head")
self.close_client(client)
return
class VPSMgr(object):
""" all exception should catch and log in this class """
VERSION = 1
def __init__(self):
self.logger = Log("vps_mgr", config=conf)
self.logger_net = Log("vps_mgr_net", config=conf)
self.logger_misc = Log("misc", config=conf)
self.logger_debug = Log("debug", config=conf)
self.host_id = conf.HOST_ID
self.vpsops = VPSOps(self.logger)
self.handlers = {
CMD.OPEN: self.__class__.vps_open,
CMD.REBOOT: self.__class__.vps_reboot,
CMD.CLOSE: self.__class__.vps_close,
CMD.OS: self.__class__.vps_reinstall_os,
CMD.UPGRADE: self.__class__.vps_upgrade,
CMD.BANDWIDTH: self.__class__.vps_set_bandwidth,
CMD.RM: self.__class__.vps_delete,
CMD.PRE_SYNC: self.__class__.vps_hot_sync,
CMD.MIGRATE: self.__class__.vps_migrate,
CMD.RESET_PW: self.__class__.vps_reset_pw,
}
self._locker = threading.Lock()
self._vps_locker = dict()
self.xenstat = XenStat()
self.timer = TimerEvents(time.time, self.logger_misc)
assert conf.MONITOR_COLLECT_INV > 0
self.last_netflow = None
self.last_diskstat = None
self.monitor_inv = conf.MONITOR_COLLECT_INV
self.last_monitor_ts = None
self.timer.add_timer(conf.MONITOR_COLLECT_INV, self.monitor_vps)
self.timer.add_timer(12 * 3600, self.refresh_host_space)
self.workers = []
self.running = False
def _try_lock_vps(self, cmd, vps_id):
self._locker.acquire()
if self._vps_locker.has_key(vps_id):
_cmd = self._vps_locker.get(vps_id)
self.logger_debug.info(
"CMD %s try to lock vps%s failed: locked by CMD %s" %
(CMD._get_name(cmd), vps_id, CMD._get_name(_cmd)))
res = False
else:
self._vps_locker[vps_id] = cmd
res = True
self._locker.release()
return res
def _unlock_vps(self, cmd, vps_id):
self._locker.acquire()
try:
_cmd = self._vps_locker.get(vps_id)
if _cmd == cmd:
del self._vps_locker[vps_id]
except KeyError:
pass
self._locker.release()
def rpc_connect(self):
rpc = SAAS_Client(self.host_id, self.logger_debug)
rpc.connect()
return rpc
def monitor_vps(self):
net_result = None
disk_result = None
try:
net_result = netflow.read_proc()
disk_devs = glob.glob("/dev/main/vps*")
if 'MAIN_DISK' in dir(conf):
disk_devs.append(conf.MAIN_DISK)
disk_result = diskstat.read_stat(disk_devs)
except Exception, e:
self.logger_misc.exception(
"cannot read netflow data from proc: %s" % (str(e)))
return
ts = time.time()
dom_map = XenStore.domain_name_id_map()
dom_names = dom_map.keys()
self.xenstat.run(dom_names)
payload = CarbonPayload()
try:
payload.append("host.cpu.%s.all" % (self.host_id), ts,
self.xenstat.total_cpu)
for dom_name in dom_names:
om = re.match("^vps(\d+)$", dom_name)
if not om:
# dom0
dom_cpu = self.xenstat.dom_dict.get(dom_name)
if dom_cpu:
payload.append("host.cpu.%s.dom0" % (self.host_id),
dom_cpu['ts'], dom_cpu['cpu_avg'])
if 'MAIN_DISK' in dir(conf) and self.last_diskstat:
t_elapse = ts - self.last_monitor_ts
v = disk_result.get(conf.MAIN_DISK)
last_v = self.last_diskstat.get(conf.MAIN_DISK)
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
payload.append("host.io.%d.ops.read" % (self.host_id),
ts, read_ops)
payload.append("host.io.%d.ops.write" % (self.host_id),
ts, write_ops)
payload.append(
"host.io.%s.traffic.read" % (self.host_id), ts,
read_byte)
payload.append(
"host.io.%s.traffic.write" % (self.host_id), ts,
write_byte)
payload.append("host.io.%s.util" % (self.host_id), ts,
util)
print conf.MAIN_DISK, read_ops, write_ops, read_byte, write_byte, util
if self.last_netflow:
t_elapse = ts - self.last_monitor_ts
v = net_result.get(conf.EXT_INF)
last_v = self.last_netflow.get(conf.EXT_INF)
_in = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_out = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_in_pp = (v[2] - last_v[2]) / t_elapse
_out_pp = (v[3] - last_v[3]) / t_elapse
payload.append(
"host.netflow.%d.ext.in" % (self.host_id), ts, _in)
payload.append(
"host.netflow.%d.ext.out" % (self.host_id), ts,
_out)
payload.append(
"host.netflow.%d.ext_pp.in" % (self.host_id), ts,
_in_pp > 0 and _in_pp or 0)
payload.append(
"host.netflow.%d.ext_pp.out" % (self.host_id), ts,
_out_pp > 0 and _out_pp or 0)
v = net_result.get(conf.INT_INF)
last_v = self.last_netflow.get(conf.INT_INF)
_in = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_out = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_in_pp = (v[2] - last_v[2]) / t_elapse
_out_pp = (v[3] - last_v[3]) / t_elapse
payload.append(
"host.netflow.%d.int.in" % (self.host_id), ts, _in)
payload.append(
"host.netflow.%d.int.out" % (self.host_id), ts,
_out)
payload.append(
"host.netflow.%d.int_pp.in" % (self.host_id), ts,
_in_pp > 0 and _in_pp or 0)
payload.append(
"host.netflow.%d.int_pp.out" % (self.host_id), ts,
_out_pp > 0 and _out_pp or 0)
else:
vps_id = int(om.group(1))
xv = self.vpsops.load_vps_meta(vps_id)
dom_cpu = self.xenstat.dom_dict.get(dom_name)
if dom_cpu:
payload.append("vps.cpu.%s" % (vps_id), dom_cpu['ts'],
dom_cpu['cpu_avg'])
if not self.last_netflow or not self.last_diskstat:
break
# net
ifname = dom_name
vif = xv.vifs.get(ifname)
v = net_result.get(ifname)
last_v = self.last_netflow.get(ifname)
t_elapse = ts - self.last_monitor_ts
if v and last_v:
# direction of vps bridged network interface needs to
# be reversed
_in = fix_flow((v[1] - last_v[1]) * 8.0 / t_elapse)
_out = fix_flow((v[0] - last_v[0]) * 8.0 / t_elapse)
_in = (vif.bandwidth and vif.bandwidth * 1024 * 1024 <
_in) and vif.bandwidth * 1024 * 1024 or _in
_out = (vif.bandwidth and vif.bandwidth * 1024 * 1024 <
_out) and vif.bandwidth * 1024 * 1024 or _out
payload.append("vps.netflow.%d.in" % (vps_id), ts, _in)
payload.append("vps.netflow.%d.out" % (vps_id), ts,
_out)
if conf.LARGE_NETFLOW and _in >= conf.LARGE_NETFLOW or _out >= conf.LARGE_NETFLOW:
self.logger_misc.warn(
"%s in: %.3f mbps, out: %.3f mbps" %
(ifname, _in / 1024.0 / 1024.0,
_out / 1024.0 / 1024.0))
# disk
if conf.USE_LVM and self.last_diskstat:
for disk in xv.data_disks.values():
v = disk_result.get(disk.dev)
last_v = self.last_diskstat.get(disk.dev)
if not last_v:
continue
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
print disk.xen_dev
payload.append(
"vps.io.%d.%s.ops.read" %
(vps_id, disk.xen_dev), ts, read_ops)
payload.append(
"vps.io.%d.%s.ops.write" %
(vps_id, disk.xen_dev), ts, write_ops)
payload.append(
"vps.io.%d.%s.traffic.read" %
(vps_id, disk.xen_dev), ts, read_byte)
payload.append(
"vps.io.%d.%s.traffic.write" %
(vps_id, disk.xen_dev), ts, write_byte)
payload.append(
"vps.io.%d.%s.util" % (vps_id, disk.xen_dev),
ts, util)
v = disk_result.get(xv.swap_store.dev)
last_v = self.last_diskstat.get(xv.swap_store.dev)
if v and last_v:
read_ops, read_byte, write_ops, write_byte, util = diskstat.cal_stat(
v, last_v, t_elapse)
payload.append(
"vps.io.%d.swap.ops.read" % (vps_id), ts,
read_ops)
payload.append(
"vps.io.%d.swap.ops.write" % (vps_id), ts,
write_ops)
payload.append(
"vps.io.%d.swap.traffic.read" % (vps_id), ts,
read_byte)
payload.append(
"vps.io.%d.swap.traffic.write" % (vps_id), ts,
write_byte)
payload.append("vps.io.%d.swap.util" % (vps_id),
ts, util)
self.last_netflow = net_result
self.last_diskstat = disk_result
self.last_monitor_ts = ts
except Exception, e:
self.logger_misc.exception(e)
return
class VpsProxy(object):
NGINX_TEMPLATE = """
server {
listen 80;
server_name %(host)s *.%(host)s;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header HOST $host.%(suffix)s;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
location / {
proxy_pass http://%(ip)s;
}
}
"""
def __init__(self):
self.logger = Log("proxy", config=config)
self.logger_rpc = Log("proxy_rpc", config=config)
self.output_path = config.NGINX_CONF_PATH
self.nginx_reload_cmd = config.NGINX_RELOAD_CMD
self.rpc = AES_RPC_Client(_private.KEY, self.logger_rpc)
def start(self):
self.is_running = True
def stop(self):
self.is_running = False
def loop(self):
while self.is_running:
self.gen_config()
time.sleep(conf.RUN_INV)
def reload_nginx(self):
subprocess.check_output(self.nginx_reload_cmd)
self.logger.info("nginx reloaded")
print "nginx reloaded"
def gen_config(self, force=False):
domain_list = None
try:
self.rpc.connect(_private.SAAS_ADDR)
try:
domain_list = self.rpc.call("proxy_domain_list")
finally:
self.rpc.close()
except Exception, e:
self.logger.exception(e)
print traceback.print_exc()
return
conf = []
for i in domain_list:
conf.append(self.NGINX_TEMPLATE % {'host':i["domain"], 'ip':i["ip"], 'suffix': _private.PROXY_DOMAIN_SUFFIX})
content = "".join(conf)
try:
if not force and os.path.exists(self.output_path) and _md5_file(self.output_path) == _md5(content):
print "skip the same"
return
f = open(self.output_path, "w")
try:
f.write(content)
finally:
f.close()
self.logger.info("conf generated")
self.reload_nginx()
except Exception, e:
self.logger.exception(e)
print traceback.print_exc()
class VpsProxy(object):
NGINX_TEMPLATE = """
server {
listen 80;
server_name %(host)s *.%(host)s;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header HOST $host.%(suffix)s;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
location / {
proxy_pass http://%(ip)s;
}
}
"""
def __init__(self):
self.logger = Log("proxy", config=config)
self.logger_rpc = Log("proxy_rpc", config=config)
self.output_path = config.NGINX_CONF_PATH
self.nginx_reload_cmd = config.NGINX_RELOAD_CMD
self.rpc = AES_RPC_Client(_private.KEY, self.logger_rpc)
def start(self):
self.is_running = True
def stop(self):
self.is_running = False
def loop(self):
while self.is_running:
self.gen_config()
time.sleep(conf.RUN_INV)
def reload_nginx(self):
subprocess.check_output(self.nginx_reload_cmd)
self.logger.info("nginx reloaded")
print "nginx reloaded"
def gen_config(self, force=False):
domain_list = None
try:
self.rpc.connect(_private.SAAS_ADDR)
try:
domain_list = self.rpc.call("proxy_domain_list")
finally:
self.rpc.close()
except Exception, e:
self.logger.exception(e)
print traceback.print_exc()
return
conf = []
for i in domain_list:
conf.append(
self.NGINX_TEMPLATE % {
'host': i["domain"],
'ip': i["ip"],
'suffix': _private.PROXY_DOMAIN_SUFFIX
})
content = "".join(conf)
try:
if not force and os.path.exists(self.output_path) and _md5_file(
self.output_path) == _md5(content):
print "skip the same"
return
f = open(self.output_path, "w")
try:
f.write(content)
finally:
f.close()
self.logger.info("conf generated")
self.reload_nginx()
except Exception, e:
self.logger.exception(e)
print traceback.print_exc()
