def add(self, args):
#check args
update = 1
count = 0
data = {}
group_list = []
if os.path.exists(self.json_file):
data = json_load(self.json_file)
if len(data) != 0:
group_list = [(k, data[k]) for k in sorted(data.keys())]
gname, v = group_list[len(group_list) - 1]
count = filter(str.isdigit, json.dumps(gname))
if args:
gname = "group" + str(int(count) + 1)
data[gname] = {}
group = data[gname]
#generate vipconf json
group['nodes'] = args.host
group['type'] = args.type
group['vip'] = args.vip
else:
update = 0
#data = sorted(data.items(), key=lambda d: d[0])
#modify vipconf about vip or nodes
if update:
json_store(data, self.config.vip_conf)
dmsg('add vipconf sucessfully')
else:
dwarn('vipconf was updated.')
def _update_version(k):
update_cmd = "echo %s > /opt/sdfs/%s/%s/status/version " % (version, k[1], k[2])
try:
dmsg('update meta %s %s.%s version to %s' % (k[0], k[1], k[2], version))
exec_remote(k[0], update_cmd)
except Exp, e:
derror("%s : %s" % (k, e))
def _get_src_nid(filename):
master_node = 0
net_info = ""
while True:
cmd = "sdfs.stat %s -v | grep master | awk '{print $7}'" % (filename)
out, err = exec_shell(cmd, p=False, need_return=True)
if len(out.split('\n')[0]) != 0:
master_node = int(out.split('\n')[0])
break
dmsg("++++++++++++++filename:%s, master:%d+++++++++++++++" % (filename, master_node))
if master_node == 0:
cmd = "sdfs.stat %s -v | grep available | awk '{print $1,$2}' | tail -n1" % (filename)
else:
cmd = "sdfs.stat %s -v | grep available | awk '{print $1,$2}' | head -n1" % (filename)
while True:
out, err = exec_shell(cmd, p=False, need_return=True)
if len(out.split('\n')[0]) != 0:
net_info = out.split('\n')[0]
break
nid = net_info.split(" ")[1].split("(")[1].split(")")[0]
return nid
def vip_confdel_group(args):
data = {}
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
if len(data) == 0:
dmsg('vipconf was null')
return
if not data.has_key(args.group):
derror("vipconfdel: %s not exist." % (args.group))
sys.exit(2)
else:
del_vips = data[args.group]["vips"]
mask = data[args.group]["mask"]
hosts = data[args.group]["nodes"].split(",")
del data[args.group]
json_store(data, json_file)
for host in hosts:
try:
_del_vip_from_host(config, host, del_vips, mask)
except Exp, e:
print e
def test_clean():
try:
os.system("/opt/sdfs/app/admin/node.py stop")
except:
pass
os.system("pkill -9 sdfs")
os.system("pkill -9 redis")
dmsg("cleanup redis")
#os.system("systemctl stop redis")
#os.system("rm -rf /var/lib/redis/*")
#os.system("systemctl start redis")
dmsg("cleanup etcd")
os.system("systemctl stop etcd")
os.system("rm -rf `grep ETCD_DATA_DIR /etc/etcd/etcd.conf | awk -F '=' '{print $2}' | sed 's/\"//g'`")
os.system("rm -rf /etc/etcd/etcd.conf")
os.system("rm -rf /var/lib/etcd/*")
os.system("chown etcd /var/lib/etcd")
try:
os.system("rm -rf %s" % (TEST_PATH))
except Exp, e:
#加try是为了规避目录非空的错误
#目录下会存在 uss.mdstat todo 查下原因
pass
def __check_volume__(self, volume):
#print volume
try:
sharding = int(
self.etcd.read("/sdfs/volume/" + volume + "/sharding").value)
replica = int(
self.etcd.read("/sdfs/volume/" + volume + "/replica").value)
#dmsg("%s sharding %d replica %d" % (volume, sharding, replica))
except etcd.EtcdKeyNotFound:
return False
lst = []
for i in range(sharding):
slot = "/sdfs/volume/%s/slot/%d/redis" % (volume, i)
try:
r = self.etcd.read(slot)._children
for i in r:
lst.append(i["key"])
except etcd.EtcdKeyNotFound:
pass
#print "sharing + replica :" + str(lst)
count = sharding * replica - len(lst)
if (count == 0):
#dmsg("%s registed" % (volume))
return False
else:
dmsg("%s need registed" % (volume))
for i in range(count):
self.__register(volume)
def bind_cache(self, cachedev, coredevs):
"""
:todo if coredev is a member of ANOTHER cset
:param cachedev:
:param coredev:
:return:
"""
force = True
self.ensure_cache_dev(cachedev, force)
for coredev in coredevs.split(','):
if self.is_coredev(coredev):
if force:
if self.is_attached_to_cache(coredev, cachedev):
if not self._check_mappingdev_exists(coredev):
raise Exp(errno.ENOENT, "%s mapping disk not exists, please check it !\n" % (coredev))
dmsg("%s already attached to cache device : %s, just ignore it !" % (coredev, cachedev))
return
else:
dmsg("dev:%s is a core device, will force destroy it!" % coredev)
self.__del_coredev_no_detach(coredev)
time.sleep(1)
else:
raise Exp(errno.EPERM, "dev:%s is already a core device, please check it" % (coredev))
self.create_coredev(coredev)
self._add_coredev_to_cache(coredev, cachedev)
def __wait_sync(self):
dbversion_local = self.__redis_dbversion()
dbversion_etcd = int(self.__etcd_get("dbversion"))
blocklist = "blocklist/" + self.hostname + ":" + self.port
blocked = None
try:
blocked = self.__etcd_get(blocklist)
except etcd.EtcdKeyNotFound:
pass
if (dbversion_etcd == dbversion_local or not blocked):
return
derror("dbversion not equal local %d remote %d" %
(dbversion_local, dbversion_etcd))
self.__etcd_set(blocklist, "blocked")
master = self.__run_slave__()
while (1):
info = self.__get_replica_info()
if (info['master_repl_offset'] == info['slave_repl_offset']):
break
dmsg("sync finish from master %s, remove block %s\n" %
s(master, blocklist))
self.__etcd_delete(blocklist)
def test():
script_path = os.path.join(CUR_PATH, 'script')
#test_objmv()
exec_shell(
"python2 %s/test_list.py --length 10 --home %s >> %s/fileop.log 2>&1" %
(CUR_PATH, TEST_PATH, LOG_PATH))
exec_shell(
"python2 %s/nfs_test.py --home %s >> %s/nfs.log 2>&1 && sync && umount /mnt/nfs"
% (script_path, TEST_PATH, LOG_PATH))
exec_shell("python2 %s/ftp_test.py --home %s >> %s/ftp.log 2>&1" %
(script_path, TEST_PATH, LOG_PATH))
#exec_shell("python2 %s/quota_test.py >> %s/misc.log 2>&1" % (script_path, LOG_PATH))
exec_shell("python2 %s/group_test.py >> %s/misc.log 2>&1" %
(script_path, LOG_PATH))
exec_shell("python2 %s/user_test.py >> %s/misc.log 2>&1" %
(script_path, LOG_PATH))
exec_shell("python2 %s/share_test.py >> %s/misc.log 2>&1" %
(script_path, LOG_PATH))
"""
"""
#exec_shell("python2 %s/fuse_test.py --home %s" % (CUR_PATH, TEST_PATH))
#exec_shell("python2 %s/flock.py" % (script_path))
#umount_nfsv4()
#time.sleep(2)
dmsg("test all successfully")
def __exit(self):
self.running = False
self.remove()
try:
dmsg("%s waiting lock thread exiting ..." % (self.workdir))
self.lockthread.join()
dmsg("lock thread exited")
except AttributeError:
pass
try:
dmsg("%s waiting mainloop thread exiting ..." % (self.workdir))
self.loop.join()
dmsg("%s mainloop thread exited" % (self.workdir))
except AttributeError:
pass
self.redis_stop()
removedir = self.workdir + "/../removed"
cmd = "mkdir -p " + removedir
print cmd
os.system(cmd)
cmd = "mv %s %s/%d.%s" % (self.workdir, removedir, self.localid,
str(uuid.uuid1()))
print cmd
os.system(cmd)
dmsg("%s exited" % (self.workdir))
def update(self):
if (self.updateable == False):
return
idx = 0
total = len(self.dict.items())
for (k, v) in self.dict.items():
idx += 1
self.dict[k] = v + str(uuid.uuid1())
for (k, v) in self.dict.items():
cmd_update = self.cmd_update(k, v)
retry = 0
while (1):
dmsg(cmd_update + '[%d/%d]' % (idx, total) + " retry %d" % (retry))
try:
test_exec(cmd_update)
except Exp, e:
if (e.errno == errno.EIO or retry > 300):
raise
time.sleep(1)
retry = retry + 1
continue
break
def __init_redis_master(self, config):
try:
self.__etcd_create("dbversion", "0")
dmsg("create dbversion " + str(self.id))
except etcd.EtcdAlreadyExist:
dmsg("dbversion exist")
pass
def __init_register_port(self, path, hostname):
prefix = "/sdfs/redis/%s" % (hostname)
if (os.path.exists(path + "/port")):
self.port_idx = int(get_value(path + "/port"))
self.port = str(self.config.redis_baseport + self.port_idx)
return True
idx = None
for i in range(NODE_PORT):
key = prefix + "/port/" + str(i)
try:
#dmsg("etcd write " + key)
self.etcd.write(key, "", prevExist=False)
idx = i
break
except etcd.EtcdAlreadyExist:
continue
if (idx == None):
derror("no port space in " + prefix)
return False
else:
dmsg("register port " + str(idx))
set_value(path + "/port", str(idx))
self.port_idx = idx
self.port = str(self.config.redis_baseport + self.port_idx)
return True
def new_test(args):
dmsg("test begin, log is in %s" % (LOG_PATH))
pool = "default"
os.system("rm -rf %s" % (LOG_PATH))
os.system("mkdir -p %s" % (LOG_PATH))
cmd = "python2 " + os.path.abspath(os.path.split(os.path.realpath(__file__))[0]) + "/docker.py create --pool %s >> %s/docker.log 2>&1" % (pool, LOG_PATH)
print cmd
os.system(cmd)
begin = time.time()
global RUNNING
RUNNING = 1
"""
if not args.nofail:
fail_simulate()
time.sleep(3)
"""
try:
test(pool)
except Exception, e:
print 'test fail'
print e
RUNNING = 0
end = time.time()
print "used %s!" % (end - begin)
kill9_self()
exit(e.errno)
def create(self):
idx = 0
total = len(self.dict.items())
for (k, v) in self.dict.items():
idx += 1
cmd_create = self.cmd_create(k, v)
cmd_remove = self.cmd_remove(k, v)
retry = 0
while (1):
dmsg(cmd_create + '[%d/%d]' % (idx, total) + " retry %d" % (retry))
try:
test_exec(cmd_create)
except Exp, e:
if (e.errno == errno.EIO or retry > 300):
raise
if (e.errno == errno.EEXIST):
if (retry == 0):
raise Exp(errno.EEXIST, "create fail, exist")
else:
try:
test_exec(cmd_remove)
except Exp, e:
pass
time.sleep(1)
retry = retry + 1
continue
break
def __register(self, volume):
workdir = None
for i in range(DISK_INSTENCE):
dir = "%s/%d" % (self.workdir, i)
if (os.path.exists(dir + "/config")):
continue
cmd = "mkdir " + dir
os.system(cmd)
workdir = dir
break
if not workdir:
return False
dmsg("workdir: " + workdir)
redis = Redisd(workdir, self.diskid, self.localid)
if not redis.init(volume):
dwarn("register %s to %s fail\n" % (workdir, volume))
return False
redis = Redisd(workdir, self.diskid, self.localid)
redis.start_loop()
self.instence.append(redis)
self.__update_instence()
return True
def _check(config, src_dir):
src_file_list = list_dir(config, src_dir)
if len(src_file_list) == 0:
dmsg("transfer ok")
return True
else:
map(print_file, src_file_list)
sys.exit(EAGAIN)
def _warp(h):
try:
dmsg("try set vip:%s on host:%s..." % (dic[h], h))
cmd = "%s setvip --vips %s --mask %s" % (config.uss_node, dic[h], mask)
exec_remote(h, cmd)
dmsg("set vip:%s on host:%s ok..." % (dic[h], h))
except Exp, e:
derror("%s : %s" % (h, e))
def test_mkdir(config):
try:
_exec_mkdir = "%s /zz" % (config.uss_mkdir)
exec_shell(_exec_mkdir)
_exec_stat = "%s /zz" % (config.uss_stat)
exec_shell(_exec_stat)
except Exp, err:
dmsg("test_mkdir or test_stat failed\n")
sys.exit(-1)
def __update_dbversion(self):
dbversion = self.__etcd_update_dbversion()
if (dbversion == -1):
dwarn("%s master fail, update dbversion\n" % (self.workdir))
return
dmsg("update dbversion to %d" % (dbversion))
cmd = "redis-cli -h %s -p %s set dbversion %d" % (self.hostname, self.port, dbversion)
os.system(cmd)
def __update_instence(self):
key = "/sdfs/redis/%s/disk/%d/instence" % (self.hostname, self.localid)
dmsg("etcd write " + key)
self.etcd.write(key, str(len(self.instence)))
key = "/sdfs/redis/%s/disk/%d/trigger" % (self.hostname, self.localid)
dmsg("etcd write " + key)
self.etcd.write(key, "0")
def move_vip(config, from_host, to_host, vip, mask):
if not _check_vip_isexist(config, from_host, vip):
dwarn("vip:%s not exist on host:%s" % (vip, from_host))
return
_del_vip_from_host(config, from_host, vip, mask)
dmsg("del vip %s from %s ok" % (vip, from_host))
_add_vip_to_host(config, to_host, vip, mask)
dmsg("add vip %s to %s ok" % (vip, to_host))
def __init_register_new__(self, slot, replica, addr):
key = "/sdfs/volume/%s/slot/%d/redis/%d" % (self.volume, slot, replica)
dmsg("set (%s, %s)" % (key, addr))
try:
#dmsg("etcd write " + key)
self.etcd.write(key, addr, prevExist=False)
#dmsg("key %s succss" % (key))
return True
except etcd.EtcdAlreadyExist:
dmsg("key %s exist" % (key))
return False
def delete_client_by_host(content, export_id, host):
if host is None:
dmsg("error : must specify host value ")
sys.exit(-1)
export_block = build_export_block(export_id, host)
block = BLOCK(export_block)
try:
new = block.del_keys(content, [])
except ArgError as e:
sys.exit(e.error)
modify_file(CONF_PATH, new)
def _start(self, ttyonly=False):
cmd = None
if self.role == "mdctl":
cmd = "%s --home %s" % (self.config.uss_mdctl, self.home)
elif self.role == "bactl":
self.start_redis()
cmd = "%s --home %s" % (self.config.uss_bactl, self.home)
elif self.role == "frctl":
#cmd = "systemctl start rpcbind"
#exec_shell(cmd)
cmd = "%s --home %s" % (self.config.uss_yfrctl, self.home)
if (self.disk_status):
derror(' * %s [disk error]' % (cmd), ttyonly)
return 1
if (self.nomount):
derror(' * %s [no mount]' % (cmd), ttyonly)
return 1
if (self.deleted):
derror(' * %s [deleted]' % (cmd), ttyonly)
return 1
if (self.skiped):
derror(' * %s [skiped]' % (cmd), ttyonly)
return 1
if (self.running()):
dwarn(' * %s [running]' % (cmd), ttyonly)
return 1
if self.config.testing and self.config.valgrind:
valgrind = "valgrind --tool=memcheck --leak-check=full --show-reachable=yes -v "
logdir = "%s/log/" % (self.config.home)
os.system("mkdir -p %s" % (logdir))
vallog = "%s/log/valgrind.%s.%s.log" % (self.config.home,
self.role, self.service)
cmd = "%s %s -f >>%s 2>&1 &" % (valgrind, cmd, vallog)
dmsg(cmd)
if (cmd == None):
derror(' * %s skip' % (self.home), ttyonly)
return
subprocess.call(cmd, shell=True, close_fds=True)
try:
self.__getpid(ttyonly)
except Exp, e:
dwarn('%s' % (e.err), ttyonly)
return e.errno
def test_touch(config):
for i in range(100):
try:
_exec_touch = "%s /zz/zz%d" % (config.uss_touch, i)
exec_shell(_exec_touch)
_exec_stat = "%s /zz/zz%d" % (config.uss_stat, i)
exec_shell(_exec_stat)
_exec_rm = "%s /zz/zz%d" % (config.uss_rm, i)
exec_shell(_exec_rm)
except Exp, err:
dmsg("test_touch or test_stat failed\n")
sys.exit(-1)
def tenant_list(self):
no_tenant = True
for f in os.listdir(MINIO_CONF_PATH):
name,port = self._split_dirname(f)
dmsg("\ttenant:%s\t\tport:%s" % (name, port))
no_tenant = False
if no_tenant:
dmsg("\tno tenant !")
return
def test_coredump():
"""
检查是否有core产生, 如果有就exit
"""
p = "%s/core/" % (TEST_PATH)
cores = os.listdir(p)
if (len(cores)):
dmsg("%s cores: %s" % (p, cores))
if cores:
#raise Exp(1, "has core %s" % str(cores))
derror("has core %s" % str(cores))
exit(errno.EPERM)
def vip_add(args):
#check args
if not is_valid_args(args):
derror('please check args')
sys.exit(1)
update = 1
count = 0
data = {}
group_list = []
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
args.host = list_distinct(args.host)
args.vip = list_distinct(args.vip)
if len(data) != 0:
group_list = [(k, data[k]) for k in sorted(data.keys())]
for k,v in group_list:
if v['nodes'] == args.host and v['vips'] == args.vip \
and v['type'] == args.type and v['mask'] == args.mask:
update = 0
break
gname, v = group_list[len(group_list) - 1]
count = filter(str.isdigit, json.dumps(gname))
if args:
gname = "group" + str(int(count)+1)
data[gname] = {}
group = data[gname]
#generate vipconf json
group['nodes'] = args.host
group['type'] = args.type
group['vips'] = args.vip
group['mask'] = args.mask
else:
update = 0
#data = sorted(data.items(), key=lambda d: d[0])
#modify vipconf about vip or nodes
if update:
json_store(data, json_file)
dmsg('add vipconf sucessfully')
else:
dwarn('vipconf was updated.')
def create(self, hosts):
#self._add_check_env(hosts)
dmsg("update ...")
self.update(hosts=hosts)
dmsg("init etcd ...")
self._init_etcd(hosts=hosts)
dmsg("init cluster config ...")
conf = ClusterConf(config=self.config, lock=False)
dmsg("add node ...")
self.config = conf.add_node(hosts)
self._init_env(hosts)
dmsg("start service ...")
self._start_service(hosts)
