def __register(self, volume):
workdir = None
for i in range(DISK_INSTENCE):
dir = "%s/%d" % (self.workdir, i)
if (os.path.exists(dir + "/config")):
continue
cmd = "mkdir " + dir
os.system(cmd)
workdir = dir
break
if not workdir:
return False
dmsg("workdir: " + workdir)
redis = Redisd(workdir, self.diskid, self.localid)
if not redis.init(volume):
dwarn("register %s to %s fail\n" % (workdir, volume))
return False
redis = Redisd(workdir, self.diskid, self.localid)
redis.start_loop()
self.instence.append(redis)
self.__update_instence()
return True
def _mount():
cmd = "mount | grep /dev/drbd0|grep /var/lib/leveldb"
try:
exec_shell(cmd)
except Exp, e:
dwarn('%s' % e)
return False
def drbd_primary():
try:
_drbd_primary_set()
except Exception, e:
_drbd_primary_prep()
_drbd_primary_set()
dwarn(e)
def add(self, args):
#check args
update = 1
count = 0
data = {}
group_list = []
if os.path.exists(self.json_file):
data = json_load(self.json_file)
if len(data) != 0:
group_list = [(k, data[k]) for k in sorted(data.keys())]
gname, v = group_list[len(group_list) - 1]
count = filter(str.isdigit, json.dumps(gname))
if args:
gname = "group" + str(int(count) + 1)
data[gname] = {}
group = data[gname]
#generate vipconf json
group['nodes'] = args.host
group['type'] = args.type
group['vip'] = args.vip
else:
update = 0
#data = sorted(data.items(), key=lambda d: d[0])
#modify vipconf about vip or nodes
if update:
json_store(data, self.config.vip_conf)
dmsg('add vipconf sucessfully')
else:
dwarn('vipconf was updated.')
def _vip_exist(eth, vip):
cmd = "ip addr|grep %s|grep %s" % (vip, eth)
try:
exec_shell(cmd, p=True)
except Exp, e:
dwarn('_vip exist %s' % e)
return False
def __load_cluster(self):
conf = os.path.join(self.home, "etc/cluster.conf")
cluster = {}
self.cluster = cluster
if not os.path.isfile(conf):
dwarn("not found %s" % (conf))
return
with open(conf, "r") as f:
for line in f.readlines():
host, redis, mond, cds, nfs, ftp = self.__parse_line(line)
cluster.update({
host: {
"redis": redis,
"mond": mond,
"cds": cds,
"nfs": nfs,
"ftp": ftp
}
})
self.cluster = cluster
#print self.cluster
try:
self.service = cluster[self.hostname]
except:
derror("node %s not found in cluster.conf" % (self.hostname))
exit(1)
def mounted(mount_point='/mnt/nfs'):
_exec = 'cat /proc/mounts | grep %s' % (mount_point)
try:
exec_shell(_exec, need_return=True, timeout=10)
except Exp, e:
dwarn("%s : %s\n" % (_exec, str(e)))
return False
def import_raid_foreign(self):
all_pdinfo = self.get_all_pdinfo()
found = False
for adp in all_pdinfo:
for pd in all_pdinfo[adp]:
if all_pdinfo[adp][pd]['stat'] == 'Unconfigured(bad)':
res = _exec_pipe(self.cmd + [
'-PDMakeGood',
'-PhysDrv[' + all_pdinfo[adp][pd]['encl'] + ':' + all_pdinfo[adp][pd]['slot'] + ']',
'-a' + adp,
'-Nolog'
], 0, False)
res = _exec_pipe(self.cmd + [
'-CfgForeign',
'-Import',
'-a' + adp,
'-Nolog'
], 0, False)
found = True
if all_pdinfo[adp][pd]['foreign'] == 'Foreign':
res = _exec_pipe(self.cmd + [
'-CfgForeign',
'-Import',
'-a' + adp,
'-Nolog'
], 0, False)
found = True
if not found:
dwarn("no Device state 'Foreign'")
def register(slist, hlist):
for ssd in slist:
_exec_echo = "echo %s > /sys/fs/bcache/register" % ssd
try:
exec_shell(_exec_echo)
except Exp, err:
dwarn('%s register failed' % (ssd))
return False
def tenant_del(self, name):
fname = _get_full_name(name, is_raise=0)
if len(fname) != 0:
cmd = "ps -ef | grep '%s ' | grep -v 'grep' | awk '{print $2}'| xargs kill -9" % (fname)
exec_shell(cmd)
cmd = "rm -rf %s/%s" % (MINIO_CONF_PATH, fname)
exec_shell(cmd)
else:
dwarn("name:%s not exists, please check it!" % name)
def list_user(self):
if self.get_user_list(self.ftp_conf):
if len(self.user_list): #//ftp.conf中已有用户配置
map(self._format, self.user_list)
else:
dwarn('empty record')
return True
else:
return False
def __update_dbversion(self):
dbversion = self.__etcd_update_dbversion()
if (dbversion == -1):
dwarn("%s master fail, update dbversion\n" % (self.workdir))
return
dmsg("update dbversion to %d" % (dbversion))
cmd = "redis-cli -h %s -p %s set dbversion %d" % (self.hostname, self.port, dbversion)
os.system(cmd)
def move_vip(config, from_host, to_host, vip, mask):
if not _check_vip_isexist(config, from_host, vip):
dwarn("vip:%s not exist on host:%s" % (vip, from_host))
return
_del_vip_from_host(config, from_host, vip, mask)
dmsg("del vip %s from %s ok" % (vip, from_host))
_add_vip_to_host(config, to_host, vip, mask)
dmsg("add vip %s to %s ok" % (vip, to_host))
def _vip_get_local(vips, local):
local = socket.gethostname()
for h in vips.keys():
if h == local:
eth = vips[h]['eth']
vip = vips[h]['vip']
return (eth, vip)
dwarn("no vip")
return (None, None)
def _vip_unset_other(vips, local):
for h in vips.keys():
if h == local:
continue
eth = vips[h]['eth']
vip = vips[h]['vip']
try:
_vip_unset_remote(h, eth, vip)
except Exp, e:
dwarn(e)
def drbd_cstate():
states = []
for h in drbd_hosts():
state = None
cmd = "drbdadm cstate mds"
try:
state, _ = exec_remote(h, cmd, exectimeout=7)
except Exp, e:
dwarn(e)
states.append(state)
def _start(self, ttyonly=False):
cmd = None
if self.role == "mdctl":
cmd = "%s --home %s" % (self.config.uss_mdctl, self.home)
elif self.role == "bactl":
self.start_redis()
cmd = "%s --home %s" % (self.config.uss_bactl, self.home)
elif self.role == "frctl":
#cmd = "systemctl start rpcbind"
#exec_shell(cmd)
cmd = "%s --home %s" % (self.config.uss_yfrctl, self.home)
if (self.disk_status):
derror(' * %s [disk error]' % (cmd), ttyonly)
return 1
if (self.nomount):
derror(' * %s [no mount]' % (cmd), ttyonly)
return 1
if (self.deleted):
derror(' * %s [deleted]' % (cmd), ttyonly)
return 1
if (self.skiped):
derror(' * %s [skiped]' % (cmd), ttyonly)
return 1
if (self.running()):
dwarn(' * %s [running]' % (cmd), ttyonly)
return 1
if self.config.testing and self.config.valgrind:
valgrind = "valgrind --tool=memcheck --leak-check=full --show-reachable=yes -v "
logdir = "%s/log/" % (self.config.home)
os.system("mkdir -p %s" % (logdir))
vallog = "%s/log/valgrind.%s.%s.log" % (self.config.home,
self.role, self.service)
cmd = "%s %s -f >>%s 2>&1 &" % (valgrind, cmd, vallog)
dmsg(cmd)
if (cmd == None):
derror(' * %s skip' % (self.home), ttyonly)
return
subprocess.call(cmd, shell=True, close_fds=True)
try:
self.__getpid(ttyonly)
except Exp, e:
dwarn('%s' % (e.err), ttyonly)
return e.errno
def __init_redis_slave(self, config):
retry = 0
while (self.running):
try:
self.__etcd_get("dbversion")
return True
except:
if (retry > 100):
dwarn("get dbversion " + str(self.id) + " fail")
return False
time.sleep(0.1)
retry = retry + 1
def vip_add(args):
#check args
if not is_valid_args(args):
derror('please check args')
sys.exit(1)
update = 1
count = 0
data = {}
group_list = []
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
args.host = list_distinct(args.host)
args.vip = list_distinct(args.vip)
if len(data) != 0:
group_list = [(k, data[k]) for k in sorted(data.keys())]
for k,v in group_list:
if v['nodes'] == args.host and v['vips'] == args.vip \
and v['type'] == args.type and v['mask'] == args.mask:
update = 0
break
gname, v = group_list[len(group_list) - 1]
count = filter(str.isdigit, json.dumps(gname))
if args:
gname = "group" + str(int(count)+1)
data[gname] = {}
group = data[gname]
#generate vipconf json
group['nodes'] = args.host
group['type'] = args.type
group['vips'] = args.vip
group['mask'] = args.mask
else:
update = 0
#data = sorted(data.items(), key=lambda d: d[0])
#modify vipconf about vip or nodes
if update:
json_store(data, json_file)
dmsg('add vipconf sucessfully')
else:
dwarn('vipconf was updated.')
def __set_ldconfig(self):
ldconfig = "/etc/ld.so.conf.d/sdfs.conf"
libs = ["%s/app/lib" % (self.home), "/usr/local/lib"]
if os.path.isfile(ldconfig):
v = get_value(ldconfig)
if not v.startswith(self.home):
dwarn("home is: %s, but ldconfig unmatched!!!" % (self.home))
else:
for lib in libs:
cmd = "echo %s >> %s" % (lib, ldconfig)
exec_shell(cmd)
exec_shell("ldconfig")
def prepare_mount_nfs(config):
if not nfs_installed():
sys.exit(-1)
# 判断nfs是否正常运行
if nfs_running():
# 判断nfs客户端是否已经mount
if not mounted():
if not start_mount_nfsv4(config):
sys.exit(-1)
else:
dwarn('nfs-ganesha != running healthy\n')
sys.exit(-1)
def __run_slave(self):
dmsg("%s run as slave" % (self.workdir))
master = self.__set_slave__()
if (master == None):
dmsg("set slave fail")
return
while (self.running):
if (self.lock.is_acquired):
dwarn("%s master %s fail\n" % (self.workdir, master))
break
else:
time.sleep(1)
def drbd_secondary():
retry = 0
retry_max = 10
while True:
try:
_drbd_secondary()
break
except Exp, e:
dwarn("wait secondary %s" % (e))
retry = retry + 1
time.sleep(1)
if (retry > retry_max):
raise Exp(1, "secondary fail")
def vip_confadd(args):
#check args
if not is_valid_ip(args):
derror('please check args')
sys.exit(1)
data = {}
new_host = []
new_vip = []
#load vipconf
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
if len(data) == 0:
print 'vipconf was null'
return
#reload new host and vip
old_host = json.dumps(data[args.group]['nodes']).strip('"').split(',')
old_vip = json.dumps(data[args.group]['vips']).strip('"').split(',')
if args.host:
new_host = [json.dumps(h.strip('"')) for h in args.host.split(',') if h.strip('"') not in old_host]
if args.vip:
new_vip = [json.dumps(v.strip('"')) for v in args.vip.split(',') if v.strip('"') not in old_vip]
#print old_host
#print new_host
if len(new_host) != 0:
new_host.extend(old_host)
data[args.group]['nodes'] = _list2str(new_host)
if len(new_vip) != 0:
new_vip.extend(old_vip)
data[args.group]['vips'] = _list2str(new_vip)
if len(new_host) != 0 or len(new_vip) != 0:
json_store(data, json_file)
dmsg('update vipconf sucessfully')
else:
dwarn('vipconf is already updated.')
def test_exec(cmd):
p = subprocess.Popen(cmd, shell=True)
try:
res = p.communicate()
ret = p.wait()
if (ret == 0):
pass
else:
msg = "exec cmd: %s, fail: %s" % (cmd, str(res))
raise Exp(ret, msg)
except KeyboardInterrupt as err:
dwarn("interupted")
p.kill()
exit(errno.EINTR)
test_coredump()
test_invalid_rw()
def _stop(self, ttyonly=False):
if (self.role == 'redis'):
return
if (self.nomount):
derror(' * %s [no mount]' % (self.cmd), ttyonly)
return False
if (self.running() == 0):
derror("%s already stopped" % self.cmd, ttyonly)
return 0
try:
self.__getpid(ttyonly)
except Exp, e:
dwarn('%s' % (e.err), ttyonly)
return e.errno
def init(self, volume):
self.volume = volume
if not self.__layout_global():
dwarn("load global layout fail")
return False
res = self.__register_get()
if (res == None):
dwarn("get register fail")
return False
dmsg("register %s to volume %s slot(%u, %u)" %
(self.workdir, self.volume, res[0], res[1]))
path = self.workdir
if (os.path.exists(path + "/config")):
derror("%s already inited" % (path))
return False
config_tmp = os.path.join(path, "config.tmp")
config = os.path.join(path, "config")
cmd = "mkdir -p " + config_tmp
#dmsg(cmd)
os.system(cmd)
if not self.__init_register_new(config_tmp, socket.gethostname(), res):
dwarn("init register fail")
return False
if not self.__init_redisconf(config_tmp, socket.gethostname()):
dwarn("init redis.conf fail")
return False
if not self.__init_redis(config_tmp):
dwarn("init redis fail")
return False
self.running = False
#dmsg("running " + str(self.running))
cmd = "mv " + config_tmp + " " + config
os.system(cmd)
self.__layout_local()
return True
def test_exec(cmd):
p = subprocess.Popen(cmd, shell=True)
try:
ret = p.wait()
stdout, stderr = p.communicate()
ret = p.returncode
if (ret == 0):
return
else:
msg = ""
msg = msg + "cmd: " + cmd
msg = msg + "\nstdout: " + str(stdout)
msg = msg + "\nstderr: " + str(stderr)
raise Exp(ret, msg)
except KeyboardInterrupt as err:
dwarn("interupted")
p.kill()
exit(errno.EINTR)
def poolid(self, pool):
cmd = "sdfs stat /%s | grep fileid | awk -F '-' '{print $2}'" % (pool)
while (1):
try:
(out, err) = exec_shell(cmd, need_return=True)
break
except Exp, e:
ret = e.errno
if (ret in [errno.EAGAIN, errno.EBUSY, errno.ENONET]):
time.sleep(1)
dwarn("get poolid fail, %s, retry" % (e))
continue
else:
if (retry > retry_max):
raise Exp(ret,
"get pool id fail: ret: %d, %s" % (ret, e))
else:
time.sleep(1)
retry = retry + 1
def __set_slave__(self):
retry = 0
while (1):
try:
master = self.__etcd_get("master")
except etcd.EtcdKeyNotFound:
dwarn("master not found, retry %d" % retry)
retry = retry + 1
if (retry > 5):
break;
else:
time.sleep(1)
continue
dmsg("%s get master %s" % (self.workdir, master))
cmd = "redis-cli -h %s -p %s SLAVEOF %s" % (self.hostname, self.port, master)
os.system(cmd)
return master
