def mount_volume(volname, mtype='glusterfs', mpoint='/mnt/glusterfs', \
mserver='', mclient='', options=''):
"""
Mount the gluster volume with specified options
Takes the volume name as mandatory argument
Returns a tuple of (returncode, stdout, stderr)
Returns (0, '', '') if already mounted
"""
global tc
if mserver == '':
mserver = tc.nodes[0]
if mclient == '':
mclient = tc.clients[0]
if options != '':
options = "-o %s" % options
if mtype == 'nfs' and options != '':
options = "%s,vers=3" % options
elif mtype == 'nfs' and options == '':
options = '-o vers=3'
ret, _, _ = tc.run(mclient, "mount | grep %s | grep %s | grep \"%s\"" \
% (volname, mpoint, mserver), verbose=False)
if ret == 0:
tc.logger.debug("Volume %s is already mounted at %s" \
% (volname, mpoint))
return (0, '', '')
mcmd = "mount -t %s %s %s:%s %s" % \
(mtype, options, mserver, volname, mpoint)
tc.run(mclient, "test -d %s || mkdir -p %s" % (mpoint, mpoint), \
verbose=False)
return tc.run(mclient, mcmd)
def gluster_basic_test():
tc.logger.info("Testing gluster volume create and mounting")
volname = tc.config_data["VOLNAME"]
mount_type = tc.config_data["MOUNT_TYPE"]
mountpoint = tc.config_data["MOUNTPOINT"]
mnode = tc.nodes[0]
client = tc.clients[0]
_rc = True
ret = setup_vol()
if not ret:
tc.logger.error("Unable to setup the volume %s" % volname)
return False
tc.run(mnode, "gluster volume status %s" % volname)
ret, _, _ = mount_volume(volname, mount_type, mountpoint, mclient=client)
if ret != 0:
tc.logger.error("mounting volume %s failed" % volname)
_rc = False
else:
ret, _, _ = tc.run(client, "cp -r /etc %s" % mountpoint)
if ret != 0:
tc.logger.error("cp failed on the mountpoint")
_rc = False
umount_volume(client, mountpoint)
ret = stop_volume(volname)
if not ret:
_rc = False
ret = delete_volume(volname)
if not ret:
_rc = False
return _rc
def calculate_checksum(file_list, server=''):
"""
This module calculates checksum (sha256sum) for the given file list
@paramater:
* file_list - <list> absolute file names for which checksum to be
calculated
* server - <str> (optional) name of the server.
If not given, the function takes the
first node from config file
@Returns: checksum value in dict format, on success
None, on failure
"""
if server == '':
server = tc.nodes[0]
cmd = "sha256sum %s" % ' '.join(file_list)
ret = tc.run(server, cmd)
if ret[0] != 0:
tc.logger.error("Failed to execute checksum command in server %s" \
% server)
return None
checksum_dict = {}
for line in ret[1].split('\n')[:-1]:
match = re.search(r'^(\S+)\s+(\S+)', line.strip())
if match is None:
tc.logger.error("checksum output is not in \
expected format")
return None
checksum_dict[match.group(2)] = match.group(1)
return checksum_dict
def snap_activate(snapname, server=''):
"""
Activate the snap and returns the output
"""
if server == '':
server = tc.nodes[0]
return tc.run(server, "gluster snapshot activate %s" % snapname)
def pool_list(pnode=''):
"""
Does pool list on the given node
Returns: On success, pool list information in list of dictionary format
On Failure, None
"""
if pnode == '':
pnode = tc.nodes[0]
ret = tc.run(pnode, "gluster pool list")
if ret[0] != 0:
tc.logger.error("Failed to execute pool list in node %s" % pnode)
return None
pool_info = []
for index, item in enumerate(ret[1].split('\n')[:-1]):
match = re.search(r'(\S+)\s*\t*\s*(\S+)\s*\t*\s*(\S+)\s*', item)
if match is not None:
if index == 0:
keys = match.groups()
else:
temp_dict = {}
for num, element in enumerate(match.groups()):
temp_dict[keys[num]] = element
pool_info.append(temp_dict)
return pool_info
def peer_probe(pnode='', servers='', timeout=10):
"""
Does peer probe and validates the same
Returns True on success and False on failure
Note: Input for parameter 'servers' should be in list format
"""
if pnode == '':
pnode = tc.nodes[0]
if servers == '':
servers = tc.nodes[1:]
for server in servers:
ret = tc.run(pnode, "gluster peer probe %s" % server)
if ret[0] != 0 or re.search(r'^peer\sprobe\:\ssuccess(.*)', ret[1]) \
is None:
tc.logger.error("Failed to do peer probe for node %s" % server)
return False
time.sleep(timeout)
#Validating whether peer probe is successful
if not validate_peer_status(pnode, servers):
tc.logger.error("peer probe validation failed")
return False
return True
def peer_detach(pnode='', servers='', force=False, timeout=10):
"""
Does peer detach and validates the same
Returns True on success and False on failure
Note: Input for parameter 'servers' should be in list format
"""
if pnode == '':
pnode = tc.nodes[0]
if servers == '':
servers = tc.nodes[1:]
for server in servers:
if force:
cmd = "gluster peer detach %s force" % server
else:
cmd = "gluster peer detach %s" % server
ret = tc.run(pnode, cmd)
if ret[0] != 0 or re.search(r'^peer\sdetach\:\ssuccess(.*)', ret[1]) \
is None:
tc.logger.error("Failed to do peer detach for node %s" % server)
return False
time.sleep(timeout)
#Validating whether peer detach is successful
if validate_peer_status(pnode, servers):
tc.logger.error("peer detach validatiom failed")
return False
return True
def set_change_detector(mvol, svol, detector, mnode='', snode=''):
"""
Sets the change detector of the geo-rep session
Returns False if the operation failed
Returns True if the operation is successfull
"""
if mnode == '':
mnode = tc.gm_nodes[0]
if snode == '':
snode = tc.gs_nodes[0]
mountbroker = ''
if tc.config_data['MOUNTBROKER'] == 'True':
mountbroker = '%s@' % tc.config_data['GEO_USER']
try:
temp = tc.change_detector
except AttributeError:
tc.change_detector = 'changelog'
if detector == tc.change_detector:
tc.logger.debug("The change detector is already set to %s" \
% detector)
return True
cmd = "gluster volum geo-replicatio %s %s%s::%s config change_detector %s" \
% (mvol, mountbroker, snode, svol, detector)
ret, _, _ = tc.run(mnode, cmd)
if ret == 0:
tc.logger.debug("Change detector successfully set to %s" % detector)
tc.change_detector = detector
return True
else:
tc.logger.error("Unable to set the change detector to %s" % detector)
return False
def get_volume_option(volname, option='all', server=''):
"""
This module gets the option values for the given volume.
@parameter:
* volname - <str> name of the volume to get status.
* option - <str> (optional) name of the volume option to get status.
If not given, the function returns all the options for
the given volume
* server - <str> (optional) name of the server to execute the volume
status command. If not given, the function takes the
first node from config file
@Returns: value for the given volume option in dict format, on success
None, on failure
"""
if server == '':
server = tc.nodes[0]
cmd = "gluster volume get %s %s" % (volname, option)
ret = tc.run(server, cmd)
if ret[0] != 0:
tc.logger.error("Failed to execute gluster volume get command")
return None
volume_option = {}
raw_output = ret[1].split("\n")
for line in raw_output[2:-1]:
match = re.search(r'^(\S+)(.*)', line.strip())
if match is None:
tc.logger.error("gluster get volume output is not in \
expected format")
return None
volume_option[match.group(1)] = match.group(2).strip()
return volume_option
def get_rebal_status(volname, server=''):
'''
This function gives rebalance status
Valid states are started/failed/in progress/completed
if the server pararmeter is empty it takes node info from config file
'''
if server == "":
server = tc.nodes[0]
status = tc.run(server, "gluster v rebalance %s status" %volname)
if status[0] != 0:
if "not started" in status[2]:
tc.logger.error("Rebalance has not started")
return ("not started", " ")
else:
tc.logger.error("error")
return ("error", " ")
else:
rebal_dict = get_rebal_dict(status[1])
if "failed" in status[1]:
tc.logger.error("Rebalance status command failed")
return ("failed", rebal_dict)
elif "in progress" in status[1]:
tc.logger.info("Rebalance is in progress")
return ("in progress", rebal_dict)
elif "completed" in status[1]:
counter = status[1].count("completed")
nnodes = get_rebal_nodes(server)
if counter == nnodes:
tc.logger.info("Rebalance is completed")
return ("completed", rebal_dict)
else:
tc.logger.error("Rebalacne has not completed on all nodes")
return ("invalid status", rebal_dict)
def snap_delete(snapname, server=''):
"""
Deletes snapshot and returns the output
"""
if server == '':
server = tc.nodes[0]
cmd = "gluster snapshot delete %s --mode=script" % snapname
return tc.run(server, cmd)
def umount_volume(client, mountpoint):
"""
unmounts the mountpoint
Returns the output of umount command
"""
cmd = "umount %s || umount -f %s || umount -l %s" \
% (mountpoint, mountpoint, mountpoint)
return tc.run(client, cmd)
def snap_delete_all(volname, server=''):
"""
Deletes one or more snaps and returns the output
"""
if server == '':
server = tc.nodes[0]
cmd = 'ret=0; for i in `gluster snapshot list %s`; do \
gluster snapshot delete $i --mode=script || ret=1; done ; exit $ret' % volname
return tc.run(server, cmd)
def setup_geo_rep_mountbroker():
"""
setup the geo-rep mountbroker session
"""
mountbroker = tc.config_data['GEO_USER']
ggroup = tc.config_data['GEO_GROUP']
slavevol = tc.config_data['SLAVEVOL']
glusterd_volfile_cmd = \
"cp /etc/glusterfs/glusterd.vol /tmp/glusterd.vol && \
sed -i '/end-volume/d' /tmp/glusterd.vol && \
sed -i '/geo-replication/d' /tmp/glusterd.vol && \
sed -i '/mountbroker/d' /tmp/glusterd.vol && \
sed -i '/rpc-auth-allow-insecure/d' /tmp/glusterd.vol && \
echo \" option mountbroker-root /var/mountbroker-root\" >> \
/tmp/glusterd.vol && \
echo \" option mountbroker-geo-replication.%s %s\" >> /tmp/glusterd.vol && \
echo \" option geo-replication-log-group %s\" >> /tmp/glusterd.vol && \
echo \" option rpc-auth-allow-insecure on\" >> /tmp/glusterd.vol && \
echo \"end-volume\" >> /tmp/glusterd.vol && \
cp /tmp/glusterd.vol /etc/glusterfs/glusterd.vol" % \
(mountbroker, slavevol, ggroup)
for slave in tc.gs_nodes:
ret = tc.add_user(slave, mountbroker, group=ggroup)
if not ret:
tc.logger.error("Unable to add %s to group" % mountbroker)
ret, _, _ = tc.run(slave, "mkdir /var/mountbroker-root")
if ret != 0:
tc.logger.error("Unable to create directory")
ret, _, _ = tc.run(slave, "chmod 0711 /var/mountbroker-root")
if ret != 0:
tc.logger.error("cannot change permission")
ret = stop_glusterd([slave])
if not ret:
tc.logger.error("gluster stop failed")
return False
ret, _, _ = tc.run(slave, glusterd_volfile_cmd)
if ret != 0:
tc.logger.error("Unable to edit the glusterd volfile")
return False
ret = start_glusterd([slave])
if not ret:
tc.logger.error("Unable to start glusterd. Exiting")
return False
return True
def bring_down_brick(volname, bindex, node=''):
"""
Kills the glusterfsd process of the particular brick
Returns True on success and False on failure
"""
global tc
if node == '':
node = tc.nodes[0]
ret, rnode, _ = tc.run(node, "gluster volume info %s | egrep \"^Brick%d:\" \
| awk '{print $2}' | awk -F : '{print $1}'" % (volname, bindex))
if ret != 0:
return False
ret, _, _ = tc.run(rnode.rstrip(), \
"pid=`cat /var/lib/glusterd/vols/%s/run/*%s_brick%d.pid` && kill -15 $pid \
|| kill -9 $pid" % (volname, volname, bindex - 1))
if ret != 0:
return False
else:
return True
def enable_quota(volname, server=''):
"""
Enables quota on the specified volume
Returns the output of quota cli command
"""
if server == '':
server = tc.nodes[0]
cmd = "gluster volume quota %s enable" % volname
return tc.run(server, cmd)
def snap_create(volname, snapname, server='', desc=''):
"""
Runs snap create command and returns the output
"""
if server == '':
server = tc.nodes[0]
if desc != '':
desc = "description %s" % desc
ret = tc.run(server, "gluster snapshot create %s %s %s" \
% (volname, snapname, desc))
return ret
def set_quota_limit(volname, path='/', limit='100GB', server=''):
"""
Sets limit-usage on the path of the specified volume to
specified limit
Returs the output of quota limit-usage command
"""
if server == '':
server = tc.nodes[0]
cmd = "gluster volume quota %s limit-usage %s %s" % (volname, path, limit)
return tc.run(server, cmd)
def add_brick(volname, nbricks, replica=1, stripe=1, peers='', mnode=''):
"""
Does the gluster add-brick. If peer is '', peers from the config
is taken. And replica/stripe will not be used by default.
Returns the output of add-brick command, which would be a tuple of
(retcode, stdout, sstderr) from gluster add-brick command.
"""
global tc
if peers == '':
peers = tc.peers[:]
if mnode == '':
mnode = tc.nodes[0]
replica = int(replica)
stripe = int(stripe)
volinfo = tc.run(mnode, "gluster volume info | egrep \"^Brick[0-9]+\"", \
verbose=False)
if volinfo[0] != 0:
tc.logger.error("Unable to get volinfo for add-brick")
return (-1, -1, -1)
bi = int(re.findall(r"%s_brick([0-9]+)" % volname, volinfo[1])[-1]) + 1
tempn = 0
n = 0
add_bricks = ''
brick_root = "/bricks"
for i in range(bi, bi + nbricks):
sn = len(re.findall(r"%s" % peers[n], volinfo[1])) + tempn
add_bricks = "%s %s:%s/brick%d/%s_brick%d" % (add_bricks, peers[n], \
brick_root, sn, volname, i)
if n < len(peers[:]) - 1:
n = n + 1
else:
n = 0
tempn = tempn + 1
repc = strc = ''
if replica != 1:
repc = "replica %d" % replica
if stripe != 1:
strc = "stripe %d" % stripe
ret = tc.run(mnode, "gluster volume add-brick %s %s %s %s" % \
(volname, repc, strc, add_bricks))
return ret
def delete_volume(volname, mnode=''):
"""
Deletes the gluster volume
Returns True if success and False if failure
"""
if mnode == '':
mnode = tc.nodes[0]
volinfo = get_volume_info(volname, mnode)
bricks = volinfo[volname]['bricks']
ret = tc.run(mnode, "gluster volume delete %s --mode=script" % volname)
if ret[0] != 0:
return False
try:
del tc.global_flag[volname]
except KeyError:
pass
for brick in bricks:
node, vol_dir = brick.split(":")
ret = tc.run(node, "rm -rf %s" % vol_dir)
return True
def get_rebal_nodes(server):
'''
This function finds out the number of rebalance nodes from
gluster v info command
Returns the number of nodes participating in rebalance process
'''
val = tc.run(server, \
"gluster v info | grep 'Brick[0-9]' | cut -d ':' -f 2 | sed 's/\ //'")
nlist = val[1].rstrip().split('\n')
nnodes = list(set(nlist))
return len(nnodes)
def start_volume(volname, mnode='', force=False):
"""
Starts the gluster volume
Returns True if success and False if failure
"""
if mnode == '':
mnode = tc.nodes[0]
frce = ''
if force:
frce = 'force'
ret = tc.run(mnode, "gluster volume start %s %s" % (volname, frce))
if ret[0] != 0:
return False
return True
def reset_volume(volname, mnode='', force=False):
"""
Reset the gluster volume
Returns True if success and False if failure
"""
if mnode == '':
mnode = tc.nodes[0]
frce = ''
if force:
frce = 'force'
ret = tc.run(mnode, "gluster volume reset %s %s --mode=script" \
% (volname, frce))
if ret[0] != 0:
return False
return True
def rebal_start(volname, server=''):
"""
Simple interface to start the gluster rebalance
@ pararmeter:
* volname
* server - defaults to tc.nodes[0]
@ returns:
True on success
False otherwise
"""
if server == '':
server = tc.nodes[0]
ret = tc.run(server, "gluster volume rebalance %s start" % volname)
if ret[0] != 0:
tc.logger.error("rebalance start %s failed" % volname)
return False
else:
tc.logger.debug("rebalance start %s successful" % volname)
return True
def snap_restore(volname, snapname, server=''):
"""
stops the volume restore the snapshot and starts the volume
Returns True upon success, False on in any step
"""
if server == '':
server = tc.nodes[0]
ret = stop_volume(volname, server)
if not ret:
return False
ret = tc.run(server, "gluster snapshot restore %s" % snapname)
if ret[0] != 0:
tc.logger.error("snapshot restore failed")
return False
ret = start_volume(volname, server)
if not ret:
return False
return True
def setup_meta_vol(servers=''):
"""
Creates, starts and mounts the gluster meta-volume on the servers
specified.
"""
if servers == '':
servers = tc.nodes
meta_volname = 'gluster_shared_storage'
mount_point = '/var/run/gluster/shared_storage'
metav_dist = int(tc.config_data['META_VOL_DIST_COUNT'])
metav_rep = int(tc.config_data['META_VOL_REP_COUNT'])
_num_bricks = metav_dist * metav_rep
repc = ''
if metav_rep > 1:
repc = "replica %d" % metav_rep
bricks = ''
brick_root = "/bricks"
_n = 0
for i in range(0, _num_bricks):
bricks = "%s %s:%s/%s_brick%d" % (bricks, servers[_n], \
brick_root, meta_volname, i)
if _n < len(servers) - 1:
_n = _n + 1
else:
_n = 0
gluster_cmd = "gluster volume create %s %s %s force" \
% (meta_volname, repc, bricks)
ret = tc.run(servers[0], gluster_cmd)
if ret[0] != 0:
tc.logger.error("Unable to create meta volume")
return False
ret = start_volume(meta_volname, servers[0])
if not ret:
tc.logger.error("Unable to start the meta volume")
return False
time.sleep(5)
for server in servers:
ret = mount_volume(meta_volname, 'glusterfs', mount_point, server, \
server)
if ret[0] != 0:
tc.logger.error("Unable to mount meta volume on %s"% server)
return False
return True
def get_volume_info(volname='all', server=''):
"""
Fetches the volume information as displayed in the volume info.
Uses xml output of volume info and parses the into to a dict
Returns a dict of dicts.
-- Volume name is the first key
-- distCount/replicaCount/Type etc are second keys
-- The value of the each second level dict depends on the key
-- For distCount/replicaCount etc the value is key
-- For bricks, the value is a list of bricks (hostname:/brick_path)
"""
if server == '':
server = tc.nodes[0]
ret = tc.run(server, "gluster volume info %s --xml" % volname, \
verbose=False)
if ret[0] != 0:
tc.logger.error("volume info returned error")
return None
root = etree.XML(ret[1])
volinfo = {}
for volume in root.findall("volInfo/volumes/volume"):
for elem in volume.getchildren():
if elem.tag == "name":
volname = elem.text
volinfo[volname] = {}
elif elem.tag == "bricks":
volinfo[volname]["bricks"] = []
for el in elem.getiterator():
if el.tag == "name":
volinfo[volname]["bricks"].append(el.text)
elif elem.tag == "options":
volinfo[volname]["options"] = {}
for option in elem.findall("option"):
for el in option.getchildren():
if el.tag == "name":
opt = el.text
if el.tag == "value":
volinfo[volname]["options"][opt] = el.text
else:
volinfo[volname][elem.tag] = elem.text
return volinfo
def get_extended_attributes_info(file_list, encoding='hex', attr_name='', server=''):
"""
This module gets extended attribute info for the given file list
@paramater:
* file_list - <list> absolute file names
* encoding - <str> (optional) encoding format
* server - <str> (optional) name of the server.
If not given, the function takes the
first node from config file
@Returns: Extended attribute info in dict format, on success
None, on failure
"""
if server == '':
server = tc.nodes[0]
server = socket.gethostbyname(server)
if attr_name == '':
cmd = "getfattr -d -m . -e %s %s" % (encoding, ' '.join(file_list))
else:
cmd = "getfattr -d -m . -n %s %s" % (attr_name, ' '.join(file_list))
ret = tc.run(server, cmd)
if ret[0] != 0:
tc.logger.error("Failed to execute getfattr command in server %s" \
% server)
return None
attr_dict = {}
for each_attr in ret[1].split('\n\n')[:-1]:
for line in each_attr.split('\n'):
if line.startswith('#'):
match = re.search(r'.*file:\s(\S+).*', line)
if match is None:
tc.logger.error("getfattr output is not in expected format")
return None
key = "/" + match.group(1)
attr_dict[key] = {}
else:
output = line.split('=')
attr_dict[key][output[0]] = output[1]
return attr_dict
def create_geo_passwordless_ssh(mnode, snode, gsuser=''):
"""
Sets up the password less ssh between two specified nodes
Returns True if successfull and False on failure
"""
if gsuser == '':
gsuser = '******'
loc = "/var/lib/glusterd/geo-replication/"
mconn = tc.get_connection(mnode, user='******')
sconn = tc.get_connection(snode, user=gsuser)
if not mconn.modules.os.path.isfile('/root/.ssh/id_rsa'):
if not mconn.modules.os.path.isfile('%s/secret.pem' % loc):
tc.logger.debug("id_rsa not present. Generating with gsec_create")
ret = tc.run(mnode, "gluster system:: execute gsec_create")
if ret[0] != 0:
tc.logger.error("Unable to generate the secret pem file")
return False
tc.logger.debug("Copying the secret.pem to id_rsa")
mconn.modules.shutil.copyfile("%s/secret.pem" % loc, \
"/root/.ssh/id_rsa")
tc.logger.debug("Copying the secret.pem.pub to id_rsa.pub")
mconn.modules.shutil.copyfile("%s/secret.pem.pub" % loc, \
"/root/.ssh/id_rsa.pub")
try:
slocal = sconn.modules.os.path.expanduser('~')
sfh = sconn.builtin.open("%s/.ssh/authorized_keys" % slocal, "a")
with mconn.builtin.open("/root/.ssh/id_rsa.pub", 'r') as f:
for line in f:
sfh.write(line)
except:
tc.logger.error("Unable to establish passwordless ssh %s@%s to %s@%s" \
% ('root', mnode, gsuser, snode))
return False
finally:
sfh.close()
mconn.close()
sconn.close()
tc.logger.debug("Password less ssh setup from %s@%s to %s@%s is %s" \
% ('root', mnode, gsuser, snode, 'successfull'))
return True
def set_sync_mode(mvol, svol, mode, mnode='', snode=''):
"""
Sets the sync mode for the geo-rep session
mastervol, slavevol and mode are mandatory parameters
slave node is optional. Will default to first slave node
master node is also optioanl. Will default to first master node
Returns True if the operation is successfull
Returns False if the operation fails
"""
if mnode == '':
mnode = tc.gm_nodes[0]
if snode == '':
snode = tc.gs_nodes[0]
if mode == 'tarssh':
cmd_string = 'use-tarssh true'
elif mode == 'rsync':
cmd_string = '\!use-tarssh'
else:
tc.logger.error("sync mode %s is not supported" % mode)
return False
mountbroker = ''
if tc.config_data['MOUNTBROKER'] == 'True':
mountbroker = '%s@' % tc.config_data['GEO_USER']
try:
temp = tc.sync_mode
except AttributeError:
tc.sync_mode = 'rsync'
if mode == tc.sync_mode:
tc.logger.debug("sync mode is already set to %s" % mode)
return True
cmd = "gluster volume geo-replication %s %s%s::%s config %s" % \
(mvol, mountbroker, snode, svol, cmd_string)
ret, _, _ = tc.run(mnode, cmd)
if ret == 0:
tc.logger.debug("sync mode set to %s was successfull" % mode)
tc.sync_mode = mode
return True
else:
tc.logger.error("Unable to set the sync mode to %s" % mode)
return False
