def set_server_preconditions(self): ''' Set any server preconditions before running the tests ''' LOG.info('Stop maintenance windows') result = self.server_ssh.run_command('avmaint sched stop --ava') LOG.info('Result: %s' % result)
def generate_cmd(self, cmd, optionlist=[]): ''' This function is used to generate command ''' if optionlist: for option in optionlist: cmd = cmd + ' ' + option LOG.info("%s" % cmd) return cmd
def mapall_root(self, cmd): ''' Execute mapall --user=root command. @cmd: The command to execute, if have special characters, need escape. ''' cmd = r'echo %s | su - root -c "ssh-agent bash -c \"ssh-add /root/.ssh/rootid;mapall --user=root \'%s\'\""' \ % (self.root_password, cmd) result = self.server_ssh.run_command(cmd) LOG.info('Run cmd: %s, result: %s' % (cmd, str(result))) return result[2]
def show_summary_result(self): ''' Show summary result of backup, restore, replicate speed. Save summary result in json format on server. ''' LOG.info('Log directory: %s' % self.log_dir) LOG.info('Data set size: %s %s' % (self.total_size, self.size_unit)) LOG.info('Backup speed: %s GB/hour' % self.backup_speed) LOG.info('Restore speed: %s GB/hour' % self.restore_speed) LOG.info('Replicate speed: %s GB/hour' % self.replicate_speed) with open(self.test_summary, 'wb') as fp: json.dump(self.summary, fp)
def check_gsan_log(self, msg): ''' Check whether the given message exist in gsan logs. @msg: The message to find in gsan logs. ''' LOG.info('Check %s in gsan log' % msg) cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;mapall "tail -1000 /data01/cur/gsan.log|egrep \\\"%s\\\""' % msg result = self.ssh.run_command(cmd) if result[2] != 0: return False else: return True
def create_directory(self, directory): ''' Create directory on remote server. @directory: Directory path. ''' cmd = 'mkdir -p %s' % directory result = self.server_ssh.run_command(cmd) if result[2] != 0: LOG.error('Failed to created directory: %s' % directory) return False LOG.info('Created directory: %s' % directory) return True
def rotate_key(self): ''' Generate a new key on KMIP server and retrieved/rotated into GSAN. ''' cmd = 'avmaint atrestencryption --akmkey --ava' std_out, std_err, ret_code = self.ssh.run_command(cmd) LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code)) if ret_code == 0: if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper(): return False return True else: return False
def wipe_workdir(self, directory): ''' Wipe work directory on remote server. @directory: Directory path. ''' cmd = 'mv %s %s_%s; mkdir -p %s' % (directory, directory, self.ts, directory) # cmd = 'rm -rf %s/*' % directory result = self.server_ssh.run_command(cmd) if result[2] != 0: LOG.error('Failed to wipe work directory: %s' % directory) return False LOG.info('Wiped work directory: %s' % directory) return True
def perf_restore(self): ''' Extract from Avamar server with some data and calculate restore rate. ''' elapsed_minutes = [] restore_dir = '/home/admin/data_set/restore' sn_list = self.avtar.get_backup_sequence_number( self.server, self.root_user, self.root_password, self.account) for index, sn in enumerate(sn_list[0:self.count]): self.dg.remove_data(restore_dir, force=True) Utils.create_directory(self.client_ssh, restore_dir) log_file = 'avtar_restore_%s.log' % sn log_file = os.path.join(self.log_dir, log_file) res = self.avtar.extract_backup(self.server, self.root_user, self.root_password, self.account, sequence_number=sn, target=restore_dir, options='--logfile=%s' % log_file) if res != 0: LOG.error('Restore file failed, sequence number %s.' % sn) return False elapsed = self.analyze_log(self.client_ssh, log_file) if elapsed is None: LOG.error('Can not get elapsed time from log file.') return False speed = self.calculate_speed(self.size, elapsed) LOG.info('[%s] Restored %s %s in %s minutes: %s GB/hour.' % (index + 1, self.size, self.size_unit, elapsed, speed)) elapsed_minutes.append(elapsed) overall_elapsed = sum(elapsed_minutes) overall_speed = self.calculate_speed(self.total_size, overall_elapsed) self.restore_elapsed = overall_elapsed self.restore_speed = overall_speed self.summary['Restore'] = { 'Type': 'Restore', 'File Size': self.size_str, 'File Count': self.count, 'Total Size': self.total_size_str, 'Test Time (mins)': overall_elapsed, 'Rate (GB/Hr)': overall_speed } LOG.info( 'Restored %s %s in %s minutes: %s GB/hour.' % (self.total_size, self.size_unit, overall_elapsed, overall_speed)) self.dg.remove_data(restore_dir, force=True) return True
def get_memory_total(self): ''' Get total memory size from /proc/meminfo, convert unit to GB ''' cmd = "cat /proc/meminfo|grep MemTotal|awk '{print $2}'" result = self.ssh.run_command(cmd) if result[2]: LOG.error('Failed with return code:%s, stdout:[%s], stderr:[%s]' % (result[2], result[0], result[1])) return -1 else: size = float(result[0]) / 1024 / 1024 LOG.info('Total memory size: %s' % size) return size
def add_ops(self, op, object_name, *args, **kwargs): ''' Add operation for task, called by taskgroup class ''' if op != "add_class" and op != "add_method" and op != "ssh_reconn": raise Exception, "Invalid op:%s ,should be add_class or add_method" % ( op) if type(object_name) == StringType: LOG.info("Add operation %s:%s" % (op, object_name)) else: LOG.info("Add operation %s" % (op)) oplist = (op, object_name, args, kwargs, None, None) self.__ops.append(oplist) return True
def run_install_akm(self, root_password, address, username, password): ''' Use installAKM.sh script installs, configures, and starts Avamar Key Manager. Return result without checking. @root_password: Avamar server root user password. @address: IP Address or FQDN and port of the kmip server. @username: User Name in client public PEM. @password: Password for the Avamar system’s private key certificate. ''' cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --username=%s\\\""' \ % (root_password, password, address, username) LOG.info('Execute command: %s' % cmd) result = self.ssh.run_command(cmd) LOG.info('Install AKM result: %s' % result) return result
def get_salts_number(self): ''' Get number of salts retrieved from KMIP server. ''' cmd = 'avmaint nodelist | grep salts' std_out, std_err, ret_code = self.ssh.run_command(cmd) LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code)) if ret_code == 0: m = re.search('nr-salts="(\d+)"/>', std_out) if m: return int(m.group(1)) else: return False else: return False
def perf_backup(self): ''' Fill the Avamar server with some data and calculate backup rate. ''' elapsed_minutes = [] for index in range(self.count): data_set = '/home/admin/data_set/backup' log_file = 'avtar_backup_%s%s_%s.log' % (self.size, self.size_unit, index) log_file = os.path.join(self.log_dir, log_file) options = '--logfile=%s --nocache=true' % log_file if self.ddr: options += ' --ddr --ddr-index=%s' % self.ddr_index self.dg.generate_file(data_set, self.size, self.size_unit, 1) res = self.avtar.create_backup(self.server, self.root_user, self.root_password, self.account, data_set, options=options) self.dg.remove_data(data_set) if res != 0: LOG.error('Backup file failed.') return False elapsed = self.analyze_log(self.client_ssh, log_file) if elapsed is None: LOG.error('Can not get elapsed time from log file.') return False speed = self.calculate_speed(self.size, elapsed) LOG.info('[%s] Backed-up %s %s in %s minutes: %s GB/hour.' % (index + 1, self.size, self.size_unit, elapsed, speed)) elapsed_minutes.append(elapsed) overall_elapsed = sum(elapsed_minutes) overall_speed = self.calculate_speed(self.total_size, overall_elapsed) self.backup_elapsed = overall_elapsed self.backup_speed = overall_speed self.summary['Backup'] = { 'Type': 'Backup', 'File Size': self.size_str, 'File Count': self.count, 'Total Size': self.total_size_str, 'Test Time (mins)': overall_elapsed, 'Rate (GB/Hr)': overall_speed } LOG.info( 'Backed-up %s %s in %s minutes: %s GB/hour.' % (self.total_size, self.size_unit, overall_elapsed, overall_speed)) return True
def get_cp_attribute(self, cp_name, attribute): ''' Get the attribute value of the specific check point. call command "avmaint lscp" @cp_name: String;The checkpoint name to be search @attribute: String;The attribute name of the specific check point. @output: String or False; The attribute value of the specific check point ''' cpobjs = self._get_all_status() if (cpobjs == False): return False for cpobj in cpobjs: LOG.info("cpobj.tag:%s" % (cpobj.tag)) if (cpobj.tag == cp_name): for name, value in vars(cpobj).items(): LOG.info("name=%s value=%s" % (name, value)) if (name == attribute): return value LOG.warn("%s: Failed to get checkpoint:%s attribute %s" % (self.__classname, cp_name, attribute)) return False
def set_password(self, password): ''' Set a new password to access the salt table in the persistent store. @password: The new salt table password. password can contain any character (ASCII as well as multibyte character sets are supported) and should be enclosed in single quotes. Empty strings are not allowed. @return: The command execution result. ''' cmd = "avmaint atrestencryption --restpassword='******' --ava" % password LOG.info('Execute command: %s' % cmd) std_out, std_err, ret_code = self.ssh.run_command(cmd) LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code)) if ret_code == 0: if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper(): return False return True else: return False
def change_password(self, root_password, cert_password): ''' Replaces the private key certificate password that Avamar Key Manager stores with a new one. @root_password: Avamar server root user password. @server: The fully qualified domain name, or IP address in dotted-quad format of external key management server. @cert_password: The new private key certificate password. ''' cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --updatepassword=%s\\\""' \ % (root_password, cert_password) LOG.info('Execute command: %s' % cmd) self.ssh.write(cmd) self.ssh.read_until_regexp('Would you like to continue?\[y/n\]') self.ssh.write('y') self.ssh.read_until_regexp('Please enter the AKM Password\:') self.ssh.write(cert_password) akm_output = self.ssh.read_until_prompt() LOG.info('Update AKM password output: %s' % akm_output) return akm_output
def check_status(self): ''' Check data-at-rest encryption status. @return: data-at-rest encryption status. ''' cmd = 'avmaint nodelist --xmlperline=9999 | grep atrestencryption' LOG.info("Execute command: %s" % cmd) std_out, std_err, ret_code = self.ssh.run_command(cmd) LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code)) if ret_code == 0: m = re.search( '<atrestencryption-status enabled="(true|false)" nr-salts="(\d+)"/>', std_out) if m: return str(m.group(1)) == 'true', int(m.group(2)) else: return False, -1 else: return False, -1
def install_certificate(self, root_password): ''' Copy certificates to remote Avamar server. @root_password: Avamar server root password, installAKM.sh should be run as root user. ''' self.remote_dir = '/usr/local/avamar/etc/akm/' self.tmp_dir = '/tmp' self.ssh.directory_should_exist(self.remote_dir) for cert in os.listdir(self.cert_dir): LOG.info('+++' + str(cert)) if cert.endswith('xml'): LOG.info('Skip') continue local_cert = os.path.join(self.cert_dir, cert) tmp_cert = os.path.join(self.tmp_dir, cert) remote_cert = os.path.join(self.remote_dir, cert) self.ssh.put_file(local_cert, destination=self.tmp_dir) cmd = 'echo %s | su - root -c "mv %s %s;chown root:root %s;chmod 444 %s"' \ % (root_password, tmp_cert, remote_cert, remote_cert, remote_cert) self.ssh.run_command(cmd)
def set_salt(self, salt): ''' Set a new salt to create secure encryption key. @salt: salt is a user-defined character string (salt), which is used to create secure encryption key. salt can contain any character (ASCII as well as multibyte character sets are supported) and should be enclosed in single quotes. Empty strings are not allowed. @return: The command execution result. ''' cmd = "avmaint atrestencryption --restsalt='%s' --ava" % salt LOG.info("Execute command: %s" % cmd) std_out, std_err, ret_code = self.ssh.run_command(cmd) LOG.info('Output: %s, Error: %s, Return code: %s' % (std_out, std_err, ret_code)) if ret_code == 0: if 'ERROR' in std_out.upper() or 'ERROR' in std_err.upper(): return False return True else: return False
def perf_replicate(self, dst_server, dst_user, dst_password): ''' Run basic replicate to destination Avamar server and calculate speed. avrepl --operation=replicate --hfsaddr=src_addr --id=root --ap=Chang3M3Now. --dstaddr=dst_addr --dstid=repluser --dstap=Chang3M3Now. --dpnname=src_addr --[replscript]dstencrypt=tls --logfile=/tmp/replicate.log account 2>&1 @dst_server: Destination server. @dst_user: Destination server repl user. @dst_password: Destination server repl user password. ''' log_file = os.path.join(self.log_dir, 'replicate.log') cmd = 'avrepl --operation=replicate --hfsaddr=%s --id=%s ' \ '--ap=%s --dstaddr=%s --dstid=%s --dstap=%s --dpnname=%s ' \ '--[replscript]dstencrypt=tls %s > %s 2>&1' \ % (self.server, self.root_user, self.root_password, dst_server, dst_user, dst_password, self.server, self.account, log_file) result = self.server_ssh.run_command(cmd) if result[2] != 0: LOG.error('Replicate to %s failed, please check log file %s.' % (dst_server, log_file)) LOG.error('Return result: %s' % str(result)) return False elapsed = self.analyze_log(self.server_ssh, log_file) if elapsed is None: LOG.error('Can not get elapsed time from log file.') return False speed = self.calculate_speed(self.total_size, elapsed) self.replicate_elapsed = elapsed self.replicate_speed = speed self.summary['Replicate'] = { 'Type': 'Replicate', 'File Size': self.size_str, 'File Count': self.count, 'Total Size': self.total_size_str, 'Test Time (mins)': elapsed, 'Rate (GB/Hr)': speed } LOG.info('Replicated %s %s in %s minutes: %s GB/hour.' % (self.total_size, self.size_unit, elapsed, speed)) return True
def wait_task_group(self): ''' Wait for all the tasks to be completed in this group @output: Boolean ''' try: for index in xrange(len(self.__tasks)): task_dict = self.__tasks[index] p = task_dict['process_objs'] p.join() task_dict['status'] = 'completed' task_dict['exit_code'] = p.exitcode self.__tasks[index] = task_dict LOG.info("task[%d] completed with exit_code=%d" % (index, task_dict['exit_code'])) LOG.log_background_messages(p.pid) return True except Exception, e: LOG.error("%s:wait_task_group failed with exception" % (self.__classname)) LOG.error(str(e)) return False
def fill_up(self, start_seed, end_seed): ''' Run avtar with 1 GB backups to an Avamar server to fill up of an Avamar server for acp performance testing. @start_seed: The seed of the *first* backup to perform. @end_seed: The seed of the *last* backup to perform. ''' if self.ddr: script = 'gc-crunch-backups-1gb-chunks7k-auto-dd.sh' else: script = 'gc-crunch-backups-1gb-chunks7k-auto.sh' script = os.path.join(self.remote_dir, 'fill-up-1.0', 'scripts', script) if self.ddr: cmd = 'eval `ssh-agent`; eval `ssh-add /home/admin/.ssh/dpnid`; %s %s %s %s 27000 %s 2>&1' \ % (script, start_seed, end_seed, self.server, self.ddr_index) else: cmd = 'eval `ssh-agent`; eval `ssh-add /home/admin/.ssh/dpnid`; %s %s %s %s 27000 2>&1' \ % (script, start_seed, end_seed, self.server) result = self.server_ssh.run_command(cmd) LOG.info('Fill up server result: %s' % result) return False if result[2] else True
def enable_akm(self, root_password, address, username, password): ''' Use installAKM.sh script installs, configures, and starts Avamar Key Manager. Check script execution result. @root_password: Avamar server root user password. @address: IP Address or FQDN and port of the kmip server. @username: User Name in client public PEM. @password: Password for the Avamar system’s private key certificate. ''' # Check whether akm is already enabled status = self.status_akm() if re.search('akm \(pid \d+\) is running', status): return True # Copy certificates to server self.install_certificate(root_password) # Check whether akm configuration file is initialized if not self.is_initial_conf(): self.initialize_conf() cmd = 'echo %s | su - root -c "ssh-agent bash -c \\\"ssh-add /root/.ssh/rootid;echo \'%s\' | installAKM.sh --ip=%s --username=%s\\\""' \ % (root_password, password, address, username) LOG.info('Execute command: %s' % cmd) result = self.ssh.run_command(cmd) LOG.info('Install AKM result: %s' % result) # Check script result if 'Successfully started AKM' not in result[0]: return False # Check akm service status status = self.status_akm() if not re.search('akm \(pid \d+\) is running', status): return False # Check configuration file akm_conf = self.get_conf() if akm_conf['Address'] != address or akm_conf['UserName'] != username: return False return True
def call_instance_method(self, instance, method, *args, **kwargs): ''' Call the method specified by install and method name with the specified arguments @instance: The instance object @method: String ,The method name in the instance object @output: The output returned by the method ''' clsname = self.__current_classname result = False try: func = getattr(instance, method, False) if func: result = func(*args, **kwargs) else: LOG.error("No method:%s for class:%s" % (method, clsname)) LOG.info("Start method successfuly:%s, result=%s" % (method, result)) except Exception, e: LOG.error("%s:Execute %s.%s %s %s failed with exception" \ %(self.__classname,clsname,method,args,kwargs)) LOG.error(str(e)) return False
def restart_dpn_with_akm_stop(self): ''' Start gsan using dpnctl. ''' cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;restart.dpn' self.ssh.write(cmd) elapsed = 0 interval = 30 timeout = 900 ret_value = True while elapsed < timeout: time.sleep(interval) elapsed += interval if self.check_gsan_log('ERROR: <0001>'): LOG.info('Found akm not available in gsan log') break else: ret_value = False cmd = "ps -ef | grep restart.dpn | grep -v grep | awk '{print $2}' | xargs kill -9" result = self.ssh.run_command(cmd) if result[2]: LOG.info('Kill restart.dpn failed: %s' % str(result)) ret_value = False return ret_value
def perf_check_account(self, account): ''' Check for client account used for avtar, create account if not exist. @account: Specifies a hierarchical LOCATION on the Avamar server. If use avtar on Avamar server, need create an account. Otherwise, account will be created once the client registered, should be domain plus client name. ''' ret = 1 self.account = account cmd = 'avmgr getu --account=%s' % account result = self.server_ssh.run_command(cmd) LOG.info('Get account result: %s' % result) # Creating client account if not exist if result[2] != 0: domains = filter(None, account.split('/')) for domain in domains[:-1]: cmd = 'avmgr newd --account=%s' % domain result = self.server_ssh.run_command(cmd) # User or account already exists if result[2] == 6: continue elif result[2] != 0: LOG.error('Create account failed, %s' % result) return False cmd = 'avmgr newm --account=%s' % account result = self.server_ssh.run_command(cmd) LOG.info('Create account result: %s' % result) ret = result[2] # Remove the existing data for the client account else: ret = self.avtar.delete_all_backups(self.server, self.root_user, self.root_password, self.account) return False if ret else True
def execute_group(self): ''' Execute all the tasks in this group @output: Boolean ''' try: for index in xrange(len(self.__tasks)): LOG.info("Executing task[%d]" % (index)) task_dict = self.__tasks[index] task_obj = task_dict['task_obj'] process_obj = self.__execute_task_back(task_obj) task_dict = { 'task_obj': task_obj, 'process_objs': process_obj, 'status': 'running', 'exit_code': 255 } self.__tasks[index] = task_dict return True except Exception, e: LOG.error("%s:execute_group failed with exception" % (self.__classname)) LOG.error(str(e)) return False
def scp_debug_script(self): ''' Copy debug script to Avamar server. ''' self.get_node_list() # Copy debug script to utility node first self.ssh.directory_should_exist(self.remote_dir) self.ssh.put_file(self.chd_dump_script, destination=self.remote_dir, mode='0755') # If multi-node, copy debug script to each data node eval_cmd = 'eval `ssh-agent`;eval `ssh-add ~/.ssh/dpnid`;' if self.is_multi: for node in self.node_list: scn_cmd = 'scn %s %s:%s' % (self.remote_script, node, self.remote_dir) cmd = eval_cmd + scn_cmd result = self.ssh.run_command(cmd) if result[2]: LOG.error('Falied to copy debug script to node %s' % node) return False LOG.info('Copy debug script to all nodes.') return True
def execute_ops(self): ''' Execute the operation list in the task,called by execute_task @output: The output returned by the last method ''' result = False try: for index in xrange(len(self.__ops)): oplist = self.__ops[index] (op, object_name, args, kwargs, l_instance, l_result) = oplist result = False if op == "ssh_reconn": LOG.info("Executing task: ssh reconnecting") ssh = object_name ssh.login() cls_instance = ssh result = True elif op == "add_class": LOG.info("Executing task: init_call_class :%s" % (object_name)) cls_instance = self.init_call_class( object_name, *args, **kwargs) if cls_instance: result = True else: LOG.info("Executing task: call mothod :%s" % (object_name)) cls_instance = self.__current_instance result = self.call_instance_method(cls_instance, object_name, *args, **kwargs) result_type = type(result) if result_type is IntType and result == 0: result = True if not result: LOG.error("%s:Execute task %s %s failed " % (self.__classname, op, object_name)) return False u_oplist = (op, object_name, args, kwargs, cls_instance, result) self.__ops[index] = u_oplist except Exception, e: LOG.error("%s:Execute execute_ops failed with exception" % (self.__classname)) LOG.error(str(e))