def getAvamarType(self):
"""
Get Avamar type, first check whether it's a physical node, if it's a AVE, check config file node.cfg
Valid types: gen4/gen4s/gen4t/vmware ave/kvm ave/hyperv ave/azure ave/aws ave
"""
if not self.__ssh:
LOG.error("No ssh connection attached")
return ""
physical_machine_check_command_list = {
"gen4": "cat /proc/cpuinfo | grep -q 5504",
"gen4s": "sudo dmidecode | grep -q S2600G",
"gen4t": "/sbin/div_cli -r 2>/dev/null| grep -q GEN4T"
}
try:
for physical_type, cmd in physical_machine_check_command_list.items(
):
res = self.__ssh.run_command(cmd)
if res[2] == 0:
return physical_type
#If not physical machines, check whether it's AVE
ave_type_file = "/usr/local/avamar/etc/node.cfg"
res = self.__ssh.run_command("ls %s" % ave_type_file)
#Check whether node.cfg exited
if res[2] == 0:
res = self.__ssh.run_command("cat %s" % ave_type_file)
if res[2] == 0:
#Try to find AVE type
node_type_content = res[0]
for line in node_type_content.split("\n"):
if re.match("ave type=\s*(.*)", line):
ave_type = re.search("ave type=\s*(.*)",
line).group(1)
return ave_type + " ave"
except Exception, e:
LOG.error(str(e))
def get_chd_files(self):
'''
Get *.chd files on all nodes, meanwhile identify new
generated stripe files since the last time.
'''
# Clear list of new stripe files
self.new_file_list = []
eval_cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;'
for node in self.node_list:
ssn_cmd = 'ssn %s \'find /data0?/cur -name "*.chd"\'' % node
cmd = eval_cmd + ssn_cmd
result = self.ssh.run_command(cmd)
if result[2]:
LOG.error('Fail to get chd file list on node %s.' % node)
raise ExecutionFailed('Fail to get chd file on %s.' % node)
for line in result[0].split('\n'):
# Ignore other files
if not line.endswith('.chd'):
continue
# Ignore existing stripe files
if (node, line) in self.file_list:
continue
# Add new stripe files into two lists
self.file_list.append((node, line))
self.new_file_list.append((node, line))
def ssh_run_command_async(ssh, cmd):
try:
ssh.start_command(cmd)
return True
except Exception, e:
LOG.warn("Execute command %s failed with Exception:%s" % (cmd, str(e)))
return False
def set_server_preconditions(self):
'''
Set any server preconditions before running the tests
'''
LOG.info('Stop maintenance windows')
result = self.server_ssh.run_command('avmaint sched stop --ava')
LOG.info('Result: %s' % result)
def ssh_run_command_error_quit(ssh, cmd):
result = ssh.run_command(cmd)
if result[2]:
LOG.error("cmd failed:%s" % (cmd))
for i in range(2):
LOG.error("result[%d] :%s" % (i, result[i]))
raise Exception, "Execute command failed:%s." % (cmd)
return result[0]
def __execute_task_func(self, task_obj):
'''
Internal function, execute task
@task_obj: the object of the specified task wich generated by the new_task()
@output: Boolean
'''
LOG.debug("Into execute_task_func")
return task_obj.execute_task()
def generate_cmd(self, cmd, optionlist=[]):
'''
This function is used to generate command
'''
if optionlist:
for option in optionlist:
cmd = cmd + ' ' + option
LOG.info("%s" % cmd)
return cmd
def mapall_root(self, cmd):
'''
Execute mapall --user=root command.
@cmd: The command to execute, if have special characters, need escape.
'''
cmd = r'echo %s | su - root -c "ssh-agent bash -c \"ssh-add /root/.ssh/rootid;mapall --user=root \'%s\'\""' \
% (self.root_password, cmd)
result = self.server_ssh.run_command(cmd)
LOG.info('Run cmd: %s, result: %s' % (cmd, str(result)))
return result[2]
def __init__(self):
'''
Constructor for TaskGroup
'''
self.__classname = self.__class__.__name__
#[
#{'task_obj','process_objs','status','exit_code'},..
#]
self.__tasks = []
manager = Manager()
msg = manager.dict()
LOG.enable_background(os.getpid(), msg)
def show_summary_result(self):
'''
Show summary result of backup, restore, replicate speed.
Save summary result in json format on server.
'''
LOG.info('Log directory: %s' % self.log_dir)
LOG.info('Data set size: %s %s' % (self.total_size, self.size_unit))
LOG.info('Backup speed: %s GB/hour' % self.backup_speed)
LOG.info('Restore speed: %s GB/hour' % self.restore_speed)
LOG.info('Replicate speed: %s GB/hour' % self.replicate_speed)
with open(self.test_summary, 'wb') as fp:
json.dump(self.summary, fp)
def check_gsan_log(self, msg):
'''
Check whether the given message exist in gsan logs.
@msg: The message to find in gsan logs.
'''
LOG.info('Check %s in gsan log' % msg)
cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;mapall "tail -1000 /data01/cur/gsan.log|egrep \\\"%s\\\""' % msg
result = self.ssh.run_command(cmd)
if result[2] != 0:
return False
else:
return True
def create_directory(self, directory):
'''
Create directory on remote server.
@directory: Directory path.
'''
cmd = 'mkdir -p %s' % directory
result = self.server_ssh.run_command(cmd)
if result[2] != 0:
LOG.error('Failed to created directory: %s' % directory)
return False
LOG.info('Created directory: %s' % directory)
return True
def rotate_key(self):
'''
Generate a new key on KMIP server and retrieved/rotated into GSAN.
'''
cmd = 'avmaint atrestencryption --akmkey --ava'
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code))
if ret_code == 0:
if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper():
return False
return True
else:
return False
def unlockcp(self, cp_name):
'''
unlock checkpoint
@cp_name: String;The check point need to be locked.
@output:Boolean;True if succeed
'''
try:
cp = AvCheckPoint(self.ssh)
result = cp.unlockcp(cp_name)
except Exception, e:
LOG.error("%s: Failed to unlock check points:%s" %
(self.__classname, cp_name, str(e)))
return False
def get_latest_cp_attribute(self, attribute):
'''
Get the lastest check point attribute. call command "avmaint cpstatus"
@attribute: String,The attribute to be searched
@output: String,The value of the attribute
'''
try:
cp = AvCheckPoint(self.ssh)
cpstatus = cp.cpstatus()
except Exception, e:
LOG.error("%s: Failed to get cpstatus:%s" %
(self.__classname, str(e)))
return False
def add_ops(self, op, object_name, *args, **kwargs):
'''
Add operation for task, called by taskgroup class
'''
if op != "add_class" and op != "add_method" and op != "ssh_reconn":
raise Exception, "Invalid op:%s ,should be add_class or add_method" % (
op)
if type(object_name) == StringType:
LOG.info("Add operation %s:%s" % (op, object_name))
else:
LOG.info("Add operation %s" % (op))
oplist = (op, object_name, args, kwargs, None, None)
self.__ops.append(oplist)
return True
def get_memory_total(self):
'''
Get total memory size from /proc/meminfo, convert unit to GB
'''
cmd = "cat /proc/meminfo|grep MemTotal|awk '{print $2}'"
result = self.ssh.run_command(cmd)
if result[2]:
LOG.error('Failed with return code:%s, stdout:[%s], stderr:[%s]' %
(result[2], result[0], result[1]))
return -1
else:
size = float(result[0]) / 1024 / 1024
LOG.info('Total memory size: %s' % size)
return size
def wipe_workdir(self, directory):
'''
Wipe work directory on remote server.
@directory: Directory path.
'''
cmd = 'mv %s %s_%s; mkdir -p %s' % (directory, directory, self.ts,
directory)
# cmd = 'rm -rf %s/*' % directory
result = self.server_ssh.run_command(cmd)
if result[2] != 0:
LOG.error('Failed to wipe work directory: %s' % directory)
return False
LOG.info('Wiped work directory: %s' % directory)
return True
def attch_ssh_avall(ssh):
'''
Attch ssh to all instances in class list
@output:None. throw exception if failed
'''
clist = [
'AvCP', 'AvGC', 'AvDpn', 'AvHfs', 'AvTier', 'AvMaint', 'AvReplication',
'AvTierPlugin'
]
instances = BuiltIn().get_library_instance(all=True)
for (cls, ins) in instances.items():
if cls in clist:
LOG.debug("Attch ssh to [%s]" % (cls))
attach_ssh_instance(ssh, ins)
def chd_dump(self, node, chd_file):
'''
Run chddump script to dump stripe content.
@node: Specifies the node, such as 0.0.
@chd_file: The file path of chd stripe.
'''
eval_cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;'
ssn_cmd = "ssn %s '%s %s'" % (node, self.remote_script, chd_file)
cmd = eval_cmd + ssn_cmd
result = self.ssh.run_command(cmd)
if result[2]:
LOG.error('Run chddump failed.')
return False
return result[0]
def remove_checkpoint(self, cp_name, force=False):
'''
Remove checkpoint,call command avmaint --ava rmcp [ --risklosingallbackups ]
@cp_name: String;The check point name to be removed
@force: Bool;Whether to Remove check point forcibly
@output: Bool;
'''
try:
cp = AvCheckPoint(self.ssh)
result = cp.remove_checkpoint(cp_name, force)
except Exception, e:
LOG.error("%s: Failed to remove check points" %
(self.__classname, str(e)))
return False
def _get_all_status(self):
'''
Private function.
Get the all of check point information. call command "avmaint lscp"
@output: False or List; Wrapper of all the checkpoint information
'''
self.__cpobjs = []
try:
cp = AvCheckPoint(self.ssh)
cpobjs = cp.lscp()
except Exception, e:
LOG.error("%s: Failed to get all check points" %
(self.__classname, str(e)))
return False
def get_conf(self):
'''
Return the content of akm.xml.
'''
self.conf_path = '/usr/local/avamar/etc/akm/akm.xml'
result = self.ssh.run_command('cat %s' % self.conf_path)
root = ET.fromstring(result[0].strip())
if root.tag != 'AkmConfig':
LOG.error('The root tag name is %s, not AkmConfig.' % root.tag)
raise Exception('Wrong format of akm.xml.')
if root.getchildren():
LOG.error('The AkmConfig node has child nodes.')
raise Exception('Wrong format of akm.xml.')
self.akm_conf = root.attrib
return self.akm_conf
def run_install_akm(self, root_password, address, username, password):
'''
Use installAKM.sh script installs, configures, and starts Avamar
Key Manager. Return result without checking.
@root_password: Avamar server root user password.
@address: IP Address or FQDN and port of the kmip server.
@username: User Name in client public PEM.
@password: Password for the Avamar system’s private key certificate.
'''
cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --username=%s\\\""' \
% (root_password, password, address, username)
LOG.info('Execute command: %s' % cmd)
result = self.ssh.run_command(cmd)
LOG.info('Install AKM result: %s' % result)
return result
def get_cp_hfscheck_attribute(self, cp_name, hfscheck_attr):
'''
Get the hfscheck attribute while checkpoint creating.
@cp_name: String;The check point need to be get attribute.
@hfscheck_attr:String;The attribute in hfscheck part
@output:Boolean;True if succeed
'''
hfscheck_obj = self.get_cp_attribute(cp_name, 'hfscheck_obj')
if (hfscheck_obj == False): return False
for name, value in vars(hfscheck_obj).items():
#print "name=%s value=%s" %(name,value)
if (name == hfscheck_attr): return value
LOG.warn("%s: Failed to get checkpoint:%s attribute %s" %
(self.__classname, cp_name, attr))
return False
def get_salts_number(self):
'''
Get number of salts retrieved from KMIP server.
'''
cmd = 'avmaint nodelist | grep salts'
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code))
if ret_code == 0:
m = re.search('nr-salts="(\d+)"/>', std_out)
if m:
return int(m.group(1))
else:
return False
else:
return False
class AvCP(AvServer):
'''
Class for wrap checkpoint related commands
'''
def __init__(self, ssh=None):
super(self.__class__, self).__init__(ssh)
self.ssh = ssh
self.__classname = self.__class__.__name__
def get_latest_cp_attribute(self, attribute):
'''
Get the lastest check point attribute. call command "avmaint cpstatus"
@attribute: String,The attribute to be searched
@output: String,The value of the attribute
'''
try:
cp = AvCheckPoint(self.ssh)
cpstatus = cp.cpstatus()
except Exception, e:
LOG.error("%s: Failed to get cpstatus:%s" %
(self.__classname, str(e)))
return False
for name, value in vars(cpstatus).items():
#print "name=%s value=%s" %(name,value)
if (name == attribute):
return value
LOG.error("Invalid attribute name for check point :%s" % (attribute))
return False
def get_taskgroup_status(self):
'''
Get taskgroup status.
@output: String; "Running","Completed","Failed"
'''
try:
status = 'Completed'
for index in xrange(len(self.__tasks)):
task_dict = self.__tasks[index]
p = task_dict['process_objs']
if p.is_alive() == True:
return 'Running'
elif p.exitcode != 0:
status = 'Failed'
except Exception, e:
LOG.error("%s:is_taskgroup_success failed with exception :%s" %
(self.__classname, str(e)))
return ''
def new_task(self):
'''
Generate a new task object
@output: the new task object
'''
try:
task_obj = Task()
#{'task_obj','process_objs','status','exit_code'},..
task_dict = {
'task_obj': task_obj,
'process_objs': None,
'status': None,
'exit_code': None
}
self.__tasks.append(task_dict)
except Exception, e:
LOG.error("%s:new_task failed with exception" % (self.__classname))
LOG.error(str(e))
return False
def get_node_list(self):
'''
Get node list of the server.
avmaint ping --xmlperline=5 | egrep "<nodelist" | grep ONLINE
<nodelist id="0.2" state="ONLINE" count="54">
<nodelist id="0.1" state="ONLINE" count="49">
<nodelist id="0.0" state="ONLINE" count="58">
'''
cmd = 'avmaint ping --xmlperline=5 | egrep "<nodelist" | grep ONLINE'
result = self.ssh.run_command(cmd)
if result[2]:
LOG.error('Fail to get node list.')
raise ExecutionFailed('Fail to get node list.')
for line in result[0].split('\n'):
m = re.search(r'id="(\d\.\d{1,2})"', line)
if m:
self.node_list.append(m.group(1))
if len(self.node_list) > 1:
self.is_multi = True