def set_server_preconditions(self):
'''
Set any server preconditions before running the tests
'''
LOG.info('Stop maintenance windows')
result = self.server_ssh.run_command('avmaint sched stop --ava')
LOG.info('Result: %s' % result)
def generate_cmd(self, cmd, optionlist=[]):
'''
This function is used to generate command
'''
if optionlist:
for option in optionlist:
cmd = cmd + ' ' + option
LOG.info("%s" % cmd)
return cmd
def mapall_root(self, cmd):
'''
Execute mapall --user=root command.
@cmd: The command to execute, if have special characters, need escape.
'''
cmd = r'echo %s | su - root -c "ssh-agent bash -c \"ssh-add /root/.ssh/rootid;mapall --user=root \'%s\'\""' \
% (self.root_password, cmd)
result = self.server_ssh.run_command(cmd)
LOG.info('Run cmd: %s, result: %s' % (cmd, str(result)))
return result[2]
def show_summary_result(self):
'''
Show summary result of backup, restore, replicate speed.
Save summary result in json format on server.
'''
LOG.info('Log directory: %s' % self.log_dir)
LOG.info('Data set size: %s %s' % (self.total_size, self.size_unit))
LOG.info('Backup speed: %s GB/hour' % self.backup_speed)
LOG.info('Restore speed: %s GB/hour' % self.restore_speed)
LOG.info('Replicate speed: %s GB/hour' % self.replicate_speed)
with open(self.test_summary, 'wb') as fp:
json.dump(self.summary, fp)
def check_gsan_log(self, msg):
'''
Check whether the given message exist in gsan logs.
@msg: The message to find in gsan logs.
'''
LOG.info('Check %s in gsan log' % msg)
cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;mapall "tail -1000 /data01/cur/gsan.log|egrep \\\"%s\\\""' % msg
result = self.ssh.run_command(cmd)
if result[2] != 0:
return False
else:
return True
def create_directory(self, directory):
'''
Create directory on remote server.
@directory: Directory path.
'''
cmd = 'mkdir -p %s' % directory
result = self.server_ssh.run_command(cmd)
if result[2] != 0:
LOG.error('Failed to created directory: %s' % directory)
return False
LOG.info('Created directory: %s' % directory)
return True
def rotate_key(self):
'''
Generate a new key on KMIP server and retrieved/rotated into GSAN.
'''
cmd = 'avmaint atrestencryption --akmkey --ava'
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code))
if ret_code == 0:
if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper():
return False
return True
else:
return False
def wipe_workdir(self, directory):
'''
Wipe work directory on remote server.
@directory: Directory path.
'''
cmd = 'mv %s %s_%s; mkdir -p %s' % (directory, directory, self.ts,
directory)
# cmd = 'rm -rf %s/*' % directory
result = self.server_ssh.run_command(cmd)
if result[2] != 0:
LOG.error('Failed to wipe work directory: %s' % directory)
return False
LOG.info('Wiped work directory: %s' % directory)
return True
def perf_restore(self):
'''
Extract from Avamar server with some data and calculate restore rate.
'''
elapsed_minutes = []
restore_dir = '/home/admin/data_set/restore'
sn_list = self.avtar.get_backup_sequence_number(
self.server, self.root_user, self.root_password, self.account)
for index, sn in enumerate(sn_list[0:self.count]):
self.dg.remove_data(restore_dir, force=True)
Utils.create_directory(self.client_ssh, restore_dir)
log_file = 'avtar_restore_%s.log' % sn
log_file = os.path.join(self.log_dir, log_file)
res = self.avtar.extract_backup(self.server,
self.root_user,
self.root_password,
self.account,
sequence_number=sn,
target=restore_dir,
options='--logfile=%s' % log_file)
if res != 0:
LOG.error('Restore file failed, sequence number %s.' % sn)
return False
elapsed = self.analyze_log(self.client_ssh, log_file)
if elapsed is None:
LOG.error('Can not get elapsed time from log file.')
return False
speed = self.calculate_speed(self.size, elapsed)
LOG.info('[%s] Restored %s %s in %s minutes: %s GB/hour.' %
(index + 1, self.size, self.size_unit, elapsed, speed))
elapsed_minutes.append(elapsed)
overall_elapsed = sum(elapsed_minutes)
overall_speed = self.calculate_speed(self.total_size, overall_elapsed)
self.restore_elapsed = overall_elapsed
self.restore_speed = overall_speed
self.summary['Restore'] = {
'Type': 'Restore',
'File Size': self.size_str,
'File Count': self.count,
'Total Size': self.total_size_str,
'Test Time (mins)': overall_elapsed,
'Rate (GB/Hr)': overall_speed
}
LOG.info(
'Restored %s %s in %s minutes: %s GB/hour.' %
(self.total_size, self.size_unit, overall_elapsed, overall_speed))
self.dg.remove_data(restore_dir, force=True)
return True
def get_memory_total(self):
'''
Get total memory size from /proc/meminfo, convert unit to GB
'''
cmd = "cat /proc/meminfo|grep MemTotal|awk '{print $2}'"
result = self.ssh.run_command(cmd)
if result[2]:
LOG.error('Failed with return code:%s, stdout:[%s], stderr:[%s]' %
(result[2], result[0], result[1]))
return -1
else:
size = float(result[0]) / 1024 / 1024
LOG.info('Total memory size: %s' % size)
return size
def add_ops(self, op, object_name, *args, **kwargs):
'''
Add operation for task, called by taskgroup class
'''
if op != "add_class" and op != "add_method" and op != "ssh_reconn":
raise Exception, "Invalid op:%s ,should be add_class or add_method" % (
op)
if type(object_name) == StringType:
LOG.info("Add operation %s:%s" % (op, object_name))
else:
LOG.info("Add operation %s" % (op))
oplist = (op, object_name, args, kwargs, None, None)
self.__ops.append(oplist)
return True
def run_install_akm(self, root_password, address, username, password):
'''
Use installAKM.sh script installs, configures, and starts Avamar
Key Manager. Return result without checking.
@root_password: Avamar server root user password.
@address: IP Address or FQDN and port of the kmip server.
@username: User Name in client public PEM.
@password: Password for the Avamar system’s private key certificate.
'''
cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --username=%s\\\""' \
% (root_password, password, address, username)
LOG.info('Execute command: %s' % cmd)
result = self.ssh.run_command(cmd)
LOG.info('Install AKM result: %s' % result)
return result
def get_salts_number(self):
'''
Get number of salts retrieved from KMIP server.
'''
cmd = 'avmaint nodelist | grep salts'
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code))
if ret_code == 0:
m = re.search('nr-salts="(\d+)"/>', std_out)
if m:
return int(m.group(1))
else:
return False
else:
return False
def perf_backup(self):
'''
Fill the Avamar server with some data and calculate backup rate.
'''
elapsed_minutes = []
for index in range(self.count):
data_set = '/home/admin/data_set/backup'
log_file = 'avtar_backup_%s%s_%s.log' % (self.size, self.size_unit,
index)
log_file = os.path.join(self.log_dir, log_file)
options = '--logfile=%s --nocache=true' % log_file
if self.ddr:
options += ' --ddr --ddr-index=%s' % self.ddr_index
self.dg.generate_file(data_set, self.size, self.size_unit, 1)
res = self.avtar.create_backup(self.server,
self.root_user,
self.root_password,
self.account,
data_set,
options=options)
self.dg.remove_data(data_set)
if res != 0:
LOG.error('Backup file failed.')
return False
elapsed = self.analyze_log(self.client_ssh, log_file)
if elapsed is None:
LOG.error('Can not get elapsed time from log file.')
return False
speed = self.calculate_speed(self.size, elapsed)
LOG.info('[%s] Backed-up %s %s in %s minutes: %s GB/hour.' %
(index + 1, self.size, self.size_unit, elapsed, speed))
elapsed_minutes.append(elapsed)
overall_elapsed = sum(elapsed_minutes)
overall_speed = self.calculate_speed(self.total_size, overall_elapsed)
self.backup_elapsed = overall_elapsed
self.backup_speed = overall_speed
self.summary['Backup'] = {
'Type': 'Backup',
'File Size': self.size_str,
'File Count': self.count,
'Total Size': self.total_size_str,
'Test Time (mins)': overall_elapsed,
'Rate (GB/Hr)': overall_speed
}
LOG.info(
'Backed-up %s %s in %s minutes: %s GB/hour.' %
(self.total_size, self.size_unit, overall_elapsed, overall_speed))
return True
def get_cp_attribute(self, cp_name, attribute):
'''
Get the attribute value of the specific check point. call command "avmaint lscp"
@cp_name: String;The checkpoint name to be search
@attribute: String;The attribute name of the specific check point.
@output: String or False; The attribute value of the specific check point
'''
cpobjs = self._get_all_status()
if (cpobjs == False): return False
for cpobj in cpobjs:
LOG.info("cpobj.tag:%s" % (cpobj.tag))
if (cpobj.tag == cp_name):
for name, value in vars(cpobj).items():
LOG.info("name=%s value=%s" % (name, value))
if (name == attribute): return value
LOG.warn("%s: Failed to get checkpoint:%s attribute %s" %
(self.__classname, cp_name, attribute))
return False
def set_password(self, password):
'''
Set a new password to access the salt table in the persistent store.
@password: The new salt table password. password can contain any
character (ASCII as well as multibyte character sets are
supported) and should be enclosed in single quotes.
Empty strings are not allowed.
@return: The command execution result.
'''
cmd = "avmaint atrestencryption --restpassword='******' --ava" % password
LOG.info('Execute command: %s' % cmd)
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' %
(std_out, std_err, ret_code))
if ret_code == 0:
if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper():
return False
return True
else:
return False
def change_password(self, root_password, cert_password):
'''
Replaces the private key certificate password that Avamar Key Manager
stores with a new one.
@root_password: Avamar server root user password.
@server: The fully qualified domain name, or IP address in
dotted-quad format of external key management server.
@cert_password: The new private key certificate password.
'''
cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --updatepassword=%s\\\""' \
% (root_password, cert_password)
LOG.info('Execute command: %s' % cmd)
self.ssh.write(cmd)
self.ssh.read_until_regexp('Would you like to continue?\[y/n\]')
self.ssh.write('y')
self.ssh.read_until_regexp('Please enter the AKM Password\:')
self.ssh.write(cert_password)
akm_output = self.ssh.read_until_prompt()
LOG.info('Update AKM password output: %s' % akm_output)
return akm_output
def check_status(self):
'''
Check data-at-rest encryption status.
@return: data-at-rest encryption status.
'''
cmd = 'avmaint nodelist --xmlperline=9999 | grep atrestencryption'
LOG.info("Execute command: %s" % cmd)
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' %
(std_out, std_err, ret_code))
if ret_code == 0:
m = re.search(
'<atrestencryption-status enabled="(true|false)" nr-salts="(\d+)"/>',
std_out)
if m:
return str(m.group(1)) == 'true', int(m.group(2))
else:
return False, -1
else:
return False, -1
def install_certificate(self, root_password):
'''
Copy certificates to remote Avamar server.
@root_password: Avamar server root password, installAKM.sh should
be run as root user.
'''
self.remote_dir = '/usr/local/avamar/etc/akm/'
self.tmp_dir = '/tmp'
self.ssh.directory_should_exist(self.remote_dir)
for cert in os.listdir(self.cert_dir):
LOG.info('+++' + str(cert))
if cert.endswith('xml'):
LOG.info('Skip')
continue
local_cert = os.path.join(self.cert_dir, cert)
tmp_cert = os.path.join(self.tmp_dir, cert)
remote_cert = os.path.join(self.remote_dir, cert)
self.ssh.put_file(local_cert, destination=self.tmp_dir)
cmd = 'echo %s | su - root -c "mv %s %s;chown root:root %s;chmod 444 %s"' \
% (root_password, tmp_cert, remote_cert, remote_cert, remote_cert)
self.ssh.run_command(cmd)
def set_salt(self, salt):
'''
Set a new salt to create secure encryption key.
@salt: salt is a user-defined character string (salt), which is
used to create secure encryption key. salt can contain
any character (ASCII as well as multibyte character sets
are supported) and should be enclosed in single quotes.
Empty strings are not allowed.
@return: The command execution result.
'''
cmd = "avmaint atrestencryption --restsalt='%s' --ava" % salt
LOG.info("Execute command: %s" % cmd)
std_out, std_err, ret_code = self.ssh.run_command(cmd)
LOG.info('Output: %s, Error: %s, Return code: %s' %
(std_out, std_err, ret_code))
if ret_code == 0:
if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper():
return False
return True
else:
return False
def perf_replicate(self, dst_server, dst_user, dst_password):
'''
Run basic replicate to destination Avamar server and calculate speed.
avrepl --operation=replicate --hfsaddr=src_addr --id=root --ap=Chang3M3Now.
--dstaddr=dst_addr --dstid=repluser --dstap=Chang3M3Now. --dpnname=src_addr
--[replscript]dstencrypt=tls --logfile=/tmp/replicate.log account 2>&1
@dst_server: Destination server.
@dst_user: Destination server repl user.
@dst_password: Destination server repl user password.
'''
log_file = os.path.join(self.log_dir, 'replicate.log')
cmd = 'avrepl --operation=replicate --hfsaddr=%s --id=%s ' \
'--ap=%s --dstaddr=%s --dstid=%s --dstap=%s --dpnname=%s ' \
'--[replscript]dstencrypt=tls %s > %s 2>&1' \
% (self.server, self.root_user, self.root_password, dst_server,
dst_user, dst_password, self.server, self.account, log_file)
result = self.server_ssh.run_command(cmd)
if result[2] != 0:
LOG.error('Replicate to %s failed, please check log file %s.' %
(dst_server, log_file))
LOG.error('Return result: %s' % str(result))
return False
elapsed = self.analyze_log(self.server_ssh, log_file)
if elapsed is None:
LOG.error('Can not get elapsed time from log file.')
return False
speed = self.calculate_speed(self.total_size, elapsed)
self.replicate_elapsed = elapsed
self.replicate_speed = speed
self.summary['Replicate'] = {
'Type': 'Replicate',
'File Size': self.size_str,
'File Count': self.count,
'Total Size': self.total_size_str,
'Test Time (mins)': elapsed,
'Rate (GB/Hr)': speed
}
LOG.info('Replicated %s %s in %s minutes: %s GB/hour.' %
(self.total_size, self.size_unit, elapsed, speed))
return True
def wait_task_group(self):
'''
Wait for all the tasks to be completed in this group
@output: Boolean
'''
try:
for index in xrange(len(self.__tasks)):
task_dict = self.__tasks[index]
p = task_dict['process_objs']
p.join()
task_dict['status'] = 'completed'
task_dict['exit_code'] = p.exitcode
self.__tasks[index] = task_dict
LOG.info("task[%d] completed with exit_code=%d" %
(index, task_dict['exit_code']))
LOG.log_background_messages(p.pid)
return True
except Exception, e:
LOG.error("%s:wait_task_group failed with exception" %
(self.__classname))
LOG.error(str(e))
return False
def fill_up(self, start_seed, end_seed):
'''
Run avtar with 1 GB backups to an Avamar server to fill up
of an Avamar server for acp performance testing.
@start_seed: The seed of the *first* backup to perform.
@end_seed: The seed of the *last* backup to perform.
'''
if self.ddr:
script = 'gc-crunch-backups-1gb-chunks7k-auto-dd.sh'
else:
script = 'gc-crunch-backups-1gb-chunks7k-auto.sh'
script = os.path.join(self.remote_dir, 'fill-up-1.0', 'scripts',
script)
if self.ddr:
cmd = 'eval `ssh-agent`; eval `ssh-add /home/admin/.ssh/dpnid`; %s %s %s %s 27000 %s 2>&1' \
% (script, start_seed, end_seed, self.server, self.ddr_index)
else:
cmd = 'eval `ssh-agent`; eval `ssh-add /home/admin/.ssh/dpnid`; %s %s %s %s 27000 2>&1' \
% (script, start_seed, end_seed, self.server)
result = self.server_ssh.run_command(cmd)
LOG.info('Fill up server result: %s' % result)
return False if result[2] else True
def enable_akm(self, root_password, address, username, password):
'''
Use installAKM.sh script installs, configures, and starts Avamar
Key Manager. Check script execution result.
@root_password: Avamar server root user password.
@address: IP Address or FQDN and port of the kmip server.
@username: User Name in client public PEM.
@password: Password for the Avamar system’s private key certificate.
'''
# Check whether akm is already enabled
status = self.status_akm()
if re.search('akm \(pid \d+\) is running', status):
return True
# Copy certificates to server
self.install_certificate(root_password)
# Check whether akm configuration file is initialized
if not self.is_initial_conf():
self.initialize_conf()
cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --username=%s\\\""' \
% (root_password, password, address, username)
LOG.info('Execute command: %s' % cmd)
result = self.ssh.run_command(cmd)
LOG.info('Install AKM result: %s' % result)
# Check script result
if 'Successfully started AKM' not in result[0]:
return False
# Check akm service status
status = self.status_akm()
if not re.search('akm \(pid \d+\) is running', status):
return False
# Check configuration file
akm_conf = self.get_conf()
if akm_conf['Address'] != address or akm_conf['UserName'] != username:
return False
return True
def call_instance_method(self, instance, method, *args, **kwargs):
'''
Call the method specified by install and method name with the specified arguments
@instance: The instance object
@method: String ,The method name in the instance object
@output: The output returned by the method
'''
clsname = self.__current_classname
result = False
try:
func = getattr(instance, method, False)
if func:
result = func(*args, **kwargs)
else:
LOG.error("No method:%s for class:%s" % (method, clsname))
LOG.info("Start method successfuly:%s, result=%s" %
(method, result))
except Exception, e:
LOG.error("%s:Execute %s.%s %s %s failed with exception" \
%(self.__classname,clsname,method,args,kwargs))
LOG.error(str(e))
return False
def restart_dpn_with_akm_stop(self):
'''
Start gsan using dpnctl.
'''
cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;restart.dpn'
self.ssh.write(cmd)
elapsed = 0
interval = 30
timeout = 900
ret_value = True
while elapsed < timeout:
time.sleep(interval)
elapsed += interval
if self.check_gsan_log('ERROR: <0001>'):
LOG.info('Found akm not available in gsan log')
break
else:
ret_value = False
cmd = "ps -ef | grep restart.dpn | grep -v grep | awk '{print $2}' | xargs kill -9"
result = self.ssh.run_command(cmd)
if result[2]:
LOG.info('Kill restart.dpn failed: %s' % str(result))
ret_value = False
return ret_value
def perf_check_account(self, account):
'''
Check for client account used for avtar, create account if not exist.
@account: Specifies a hierarchical LOCATION on the Avamar server.
If use avtar on Avamar server, need create an account.
Otherwise, account will be created once the client
registered, should be domain plus client name.
'''
ret = 1
self.account = account
cmd = 'avmgr getu --account=%s' % account
result = self.server_ssh.run_command(cmd)
LOG.info('Get account result: %s' % result)
# Creating client account if not exist
if result[2] != 0:
domains = filter(None, account.split('/'))
for domain in domains[:-1]:
cmd = 'avmgr newd --account=%s' % domain
result = self.server_ssh.run_command(cmd)
# User or account already exists
if result[2] == 6:
continue
elif result[2] != 0:
LOG.error('Create account failed, %s' % result)
return False
cmd = 'avmgr newm --account=%s' % account
result = self.server_ssh.run_command(cmd)
LOG.info('Create account result: %s' % result)
ret = result[2]
# Remove the existing data for the client account
else:
ret = self.avtar.delete_all_backups(self.server, self.root_user,
self.root_password,
self.account)
return False if ret else True
def execute_group(self):
'''
Execute all the tasks in this group
@output: Boolean
'''
try:
for index in xrange(len(self.__tasks)):
LOG.info("Executing task[%d]" % (index))
task_dict = self.__tasks[index]
task_obj = task_dict['task_obj']
process_obj = self.__execute_task_back(task_obj)
task_dict = {
'task_obj': task_obj,
'process_objs': process_obj,
'status': 'running',
'exit_code': 255
}
self.__tasks[index] = task_dict
return True
except Exception, e:
LOG.error("%s:execute_group failed with exception" %
(self.__classname))
LOG.error(str(e))
return False
def scp_debug_script(self):
'''
Copy debug script to Avamar server.
'''
self.get_node_list()
# Copy debug script to utility node first
self.ssh.directory_should_exist(self.remote_dir)
self.ssh.put_file(self.chd_dump_script,
destination=self.remote_dir,
mode='0755')
# If multi-node, copy debug script to each data node
eval_cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;'
if self.is_multi:
for node in self.node_list:
scn_cmd = 'scn %s %s:%s' % (self.remote_script, node,
self.remote_dir)
cmd = eval_cmd + scn_cmd
result = self.ssh.run_command(cmd)
if result[2]:
LOG.error('Falied to copy debug script to node %s' % node)
return False
LOG.info('Copy debug script to all nodes.')
return True
def execute_ops(self):
'''
Execute the operation list in the task,called by execute_task
@output: The output returned by the last method
'''
result = False
try:
for index in xrange(len(self.__ops)):
oplist = self.__ops[index]
(op, object_name, args, kwargs, l_instance, l_result) = oplist
result = False
if op == "ssh_reconn":
LOG.info("Executing task: ssh reconnecting")
ssh = object_name
ssh.login()
cls_instance = ssh
result = True
elif op == "add_class":
LOG.info("Executing task: init_call_class :%s" %
(object_name))
cls_instance = self.init_call_class(
object_name, *args, **kwargs)
if cls_instance: result = True
else:
LOG.info("Executing task: call mothod :%s" % (object_name))
cls_instance = self.__current_instance
result = self.call_instance_method(cls_instance,
object_name, *args,
**kwargs)
result_type = type(result)
if result_type is IntType and result == 0:
result = True
if not result:
LOG.error("%s:Execute task %s %s failed " %
(self.__classname, op, object_name))
return False
u_oplist = (op, object_name, args, kwargs, cls_instance,
result)
self.__ops[index] = u_oplist
except Exception, e:
LOG.error("%s:Execute execute_ops failed with exception" %
(self.__classname))
LOG.error(str(e))