def verify_usb(conn_ssh):
if conn_ssh is None:
conn_ssh = ControllerClient.get_active_controller()
if conn_ssh:
LOG.tc_step(
"Checking if a USB flash drive with cloned image file is plugged in... "
)
usb_device_name = install_helper.get_usb_device_name(con_ssh=conn_ssh)
assert usb_device_name, "No USB found "
LOG.tc_step(
"USB flash drive found, checking for cloned image iso file ... ")
cmd = "mount | grep {} | awk \' {{ print $3}}\'".format(
usb_device_name + " ")
mount_point = conn_ssh.exec_cmd(cmd)[1]
if not mount_point:
LOG.info("Mounting USB device to /media/sysadmin ....")
install_helper.mount_usb(usb_device_name,
mount="/media/sysadmin",
con_ssh=conn_ssh)
rc, output = conn_ssh.exec_sudo_cmd("ls /media/sysadmin")
clone_files = ['boot.cat', 'clone-archive', 'install_clone']
if rc == 0 and output:
if all(f in output for f in clone_files):
return True
else:
LOG.info(
"Plugged USB {} does not appear to contain cloned image iso file"
.format(usb_device_name))
return False
else:
LOG.info(
" SSH connection with controller-0 is not available; USB cannot be checked ...."
)
return False
def pre_system_clone():
lab = InstallVars.get_install_var('LAB')
LOG.info("Preparing lab for system clone....")
if 'compute_nodes' in lab.keys() or 'storage_nodes' in lab.keys():
skip(
"The system {} is not All-in-one; clone is supported only for AIO systems"
.format(lab))
assert system_helper.get_active_controller_name(
) == 'controller-0', "controller-0 is not the active controller"
LOG.fixture_step(
"Checking if a USB flash drive is plugged in controller-0 node... ")
usb_device = install_helper.get_usb_device_name()
assert usb_device, "No USB found in controller-0"
usb_size = install_helper.get_usb_disk_size(usb_device)
LOG.info("Size of {} = {}".format(usb_device, usb_size))
if not (usb_size >= 5):
skip("Insufficient size in {} which is {}; at least 8G is required.".
format(usb_device, usb_size))
def pre_system_backup():
"""
Actions before system backup, including:
- check the USB device is ready if it is the destination
- create folder for the backup files on destination server
- collect logs on the current system
Args:
Returns:
"""
lab = InstallVars.get_install_var('LAB')
LOG.info("Preparing lab for system backup....")
backup_dest = BackupVars.get_backup_var("BACKUP_DEST")
NATBoxClient.set_natbox_client()
_backup_info = {
'backup_dest': backup_dest,
'usb_parts_info': None,
'backup_dest_full_path': None,
'dest_server': None
}
if backup_dest == 'usb':
_backup_info['dest'] = 'usb'
active_controller_name = system_helper.get_active_controller_name()
if active_controller_name != 'controller-0':
msg = "controller-0 is not the active controller"
LOG.info(msg + ", try to swact the host")
host_helper.swact_host(active_controller_name)
active_controller_name = system_helper.get_active_controller_name()
assert active_controller_name == 'controller-0', msg
LOG.fixture_step(
"Checking if a USB flash drive is plugged in controller-0 node... "
)
usb_device = install_helper.get_usb_device_name()
assert usb_device, "No USB found in controller-0"
parts_info = install_helper.get_usb_device_partition_info(
usb_device=usb_device)
part1 = "{}1".format(usb_device)
part2 = "{}2".format(usb_device)
if len(parts_info) < 3:
skip(
"USB {} is not partitioned; Create two partitions using fdisk; partition 1 = {}1, "
"size = 2G, bootable; partition 2 = {}2, size equal to the avaialble space."
.format(usb_device, usb_device, usb_device))
devices = parts_info.keys()
LOG.info("Size of {} = {}".format(
part1, install_helper.get_usb_partition_size(part1)))
if not (part1 in devices
and install_helper.get_usb_partition_size(part1) >= 2):
skip("Insufficient size in {}; at least 2G is required. {}".format(
part1, parts_info))
if not (part2 in devices
and install_helper.get_usb_partition_size(part2) >= 10):
skip("Insufficient size in {}; at least 2G is required. {}".format(
part1, parts_info))
if not install_helper.mount_usb(part2):
skip("Fail to mount USB for backups")
LOG.tc_step("Erasing existing files from USB ... ")
assert install_helper.delete_backup_files_from_usb(
part2), "Fail to erase existing file from USB"
_backup_info['usb_parts_info'] = parts_info
_backup_info['backup_dest_full_path'] = BackupRestore.USB_BACKUP_PATH
elif backup_dest == 'local':
_backup_info['dest'] = 'local'
# save backup files in Test Server which local
backup_dest_path = BackupVars.get_backup_var('BACKUP_DEST_PATH')
backup_dest_full_path = '{}/{}'.format(backup_dest_path,
lab['short_name'])
# ssh to test server
test_server_attr = dict()
test_server_attr['name'] = TestFileServer.get_hostname().split('.')[0]
test_server_attr['server_ip'] = TestFileServer.get_server()
test_server_attr['prompt'] = r'\[{}@{} {}\]\$ '\
.format(TestFileServer.get_user(), test_server_attr['name'], TestFileServer.get_user())
test_server_conn = install_helper.establish_ssh_connection(
test_server_attr['name'],
user=TestFileServer.get_user(),
password=TestFileServer.get_password(),
initial_prompt=test_server_attr['prompt'])
test_server_conn.set_prompt(test_server_attr['prompt'])
test_server_conn.deploy_ssh_key(install_helper.get_ssh_public_key())
test_server_attr['ssh_conn'] = test_server_conn
test_server_obj = Server(**test_server_attr)
_backup_info['dest_server'] = test_server_obj
# test if backup path for the lab exist in Test server
if test_server_conn.exec_cmd(
"test -e {}".format(backup_dest_full_path))[0]:
test_server_conn.exec_cmd(
"mkdir -p {}".format(backup_dest_full_path))
# delete any existing files
test_server_conn.exec_cmd("rm -rf {}/*".format(backup_dest_full_path))
_backup_info['usb_parts_info'] = None
_backup_info['backup_dest_full_path'] = backup_dest_full_path
collect_logs('before_br')
_backup_info['is_storage_lab'] = (len(system_helper.get_storage_nodes()) >
0)
return _backup_info
def pre_restore_checkup():
"""
Fixture to check the system states before doing system restore, including:
- collect logs
- check if backup files exist on the backup media
- check if the build-ids match with each other
- wipe disks
Args:
Return:
backup files:
- the backup files to restore with
"""
lab = InstallVars.get_install_var('LAB')
LOG.info("Lab info; {}".format(lab))
backup_build_id = RestoreVars.get_restore_var("BACKUP_BUILD_ID")
controller_node = lab['controller-0']
backup_src = RestoreVars.get_restore_var('backup_src'.upper())
backup_src_path = RestoreVars.get_restore_var('backup_src_path'.upper())
tis_backup_files = []
extra_controller_prompt = Prompt.TIS_NODE_PROMPT_BASE.format(lab['name'].
split('_')[0]) + '|' + \
Prompt.CONTROLLER_0
controller_conn = install_helper.ssh_to_controller(
controller_node.host_ip,
initial_prompt=extra_controller_prompt,
fail_ok=True)
LOG.info('Collect logs before restore')
if controller_conn:
collect_logs(controller_conn)
ControllerClient.set_active_controller(controller_conn)
else:
LOG.info('Cannot collect logs because no ssh connection to the lab')
if not controller_conn:
LOG.warn(
'failed to collect logs because no ssh connection established to '
'controller-0 of lab:{}'.format(controller_node.host_ip))
else:
pass
LOG.info('backup_src={}, backup_src_path={}'.format(
backup_src, backup_src_path))
if backup_src.lower() == 'usb':
if controller_conn:
LOG.info("Connection established with controller-0 ....")
ControllerClient.set_active_controller(ssh_client=controller_conn)
LOG.info(
"Checking if a USB flash drive with backup files is plugged in... "
)
usb_device_name = install_helper.get_usb_device_name(
con_ssh=controller_conn)
assert usb_device_name, "No USB found "
LOG.info("USB flash drive found, checking for backup files ... ")
usb_part_info = install_helper.get_usb_device_partition_info(
usb_device=usb_device_name, con_ssh=controller_conn)
assert usb_part_info and len(
usb_part_info) > 0, "No USB or partition found"
usb_part_name = "{}2".format(usb_device_name)
assert usb_part_name in usb_part_info.keys(
), "No {} partition exist in USB"
result, mount_point = install_helper.is_usb_mounted(
usb_device=usb_part_name, con_ssh=controller_conn)
if not result:
assert install_helper.mount_usb(usb_device=usb_part_name, con_ssh=controller_conn), \
"Unable to mount USB partition {}".format(usb_part_name)
tis_backup_files = install_helper.get_titanium_backup_filenames_usb(
usb_device=usb_part_name, con_ssh=controller_conn)
assert len(
tis_backup_files) >= 2, "Missing backup files: {}".format(
tis_backup_files)
# extract build id from the file name
file_parts = tis_backup_files[0].split('_')
file_backup_build_id = '_'.join([file_parts[3], file_parts[4]])
assert re.match(TIS_BLD_DIR_REGEX,
file_backup_build_id), " Invalid build id format {} extracted from " \
"backup_file {}".format(
file_backup_build_id, tis_backup_files[0])
if backup_build_id is not None:
if backup_build_id != file_backup_build_id:
LOG.info(
" The build id extracted from backup file is different than "
"specified; Using the extracted build id {} ....".
format(file_backup_build_id))
backup_build_id = file_backup_build_id
else:
backup_build_id = file_backup_build_id
RestoreVars.set_restore_var(backup_build_id=backup_build_id)
else:
LOG.info(" SSH connection not available yet with controller-0; "
"USB will be checked after controller boot ....")
else:
test_server_attr = dict()
test_server_attr['name'] = TestFileServer.get_hostname().split('.')[0]
test_server_attr['server_ip'] = TestFileServer.get_server()
test_server_attr['prompt'] = r'\[{}@{} {}\]\$ ' \
.format(TestFileServer.get_user(), test_server_attr['name'], TestFileServer.get_user())
test_server_conn = install_helper.establish_ssh_connection(
test_server_attr['name'],
user=TestFileServer.get_user(),
password=TestFileServer.get_password(),
initial_prompt=test_server_attr['prompt'])
test_server_conn.set_prompt(test_server_attr['prompt'])
test_server_conn.deploy_ssh_key(install_helper.get_ssh_public_key())
test_server_attr['ssh_conn'] = test_server_conn
test_server_obj = Server(**test_server_attr)
RestoreVars.set_restore_var(backup_src_server=test_server_obj)
# test if backup path for the lab exist in Test server
if os.path.basename(backup_src_path) != lab['short_name']:
backup_src_path += '/{}'.format(lab['short_name'])
RestoreVars.set_restore_var(backup_src_path=backup_src_path)
assert not test_server_conn.exec_cmd("test -e {}".format(backup_src_path))[0], \
"Missing backup files from source {}: {}".format(test_server_attr['name'],
backup_src_path)
tis_backup_files = install_helper.get_backup_files(
TITANIUM_BACKUP_FILE_PATTERN, backup_src_path, test_server_conn)
assert len(tis_backup_files) >= 2, "Missing backup files: {}".format(
tis_backup_files)
# extract build id from the file name
file_parts = tis_backup_files[0].split('_')
file_backup_build_id = '_'.join([file_parts[3], file_parts[4]])
assert re.match(TIS_BLD_DIR_REGEX,
file_backup_build_id), "Invalid build id format {} extracted from " \
"backup_file {}".format(
file_backup_build_id, tis_backup_files[0])
if backup_build_id is not None:
if backup_build_id != file_backup_build_id:
LOG.info(
" The build id extracted from backup file is different than specified; "
"Using the extracted build id {} ....".format(
file_backup_build_id))
backup_build_id = file_backup_build_id
else:
backup_build_id = file_backup_build_id
RestoreVars.set_restore_var(backup_build_id=backup_build_id)
if controller_conn:
# Wipe disks in order to make controller-0 NOT boot from hard-disks
# hosts = [k for k , v in lab.items() if isinstance(v, node.Node)]
# install_helper.wipe_disk_hosts(hosts)
if not RestoreVars.get_restore_var('skip_reinstall'):
LOG.info('Try to do wipedisk_via_helper on controller-0')
install_helper.wipedisk_via_helper(controller_conn)
assert backup_build_id, "The Build id of the system backup must be provided."
return tis_backup_files
def restore_setup(pre_restore_checkup):
"""
Fixture to do preparation before system restore.
Args:
pre_restore_checkup:
- actions done prior to this
Returen:
a dictionary
- containing infromation about target system, output directory,
build server and backup files.
"""
LOG.debug('Restore with settings:\n{}'.format(
RestoreVars.get_restore_vars()))
lab = InstallVars.get_install_var('LAB')
LOG.info("Lab info; {}".format(lab))
hostnames = [k for k, v in lab.items() if isinstance(v, node.Node)]
LOG.info("Lab hosts; {}".format(hostnames))
backup_build_id = RestoreVars.get_restore_var("BACKUP_BUILD_ID")
output_dir = ProjVar.get_var('LOG_DIR')
controller_node = lab['controller-0']
controller_prompt = ''
extra_controller_prompt = Prompt.TIS_NODE_PROMPT_BASE.format(lab['name'].split('_')[0]) + '|' + \
Prompt.CONTROLLER_0
if RestoreVars.get_restore_var('skip_reinstall'):
LOG.info('Skip reinstall as instructed')
LOG.info('Connect to controller-0 now')
controller_node.ssh_conn = install_helper.ssh_to_controller(
controller_node.host_ip,
initial_prompt=extra_controller_prompt,
fail_ok=True)
bld_server_obj = None
else:
# bld_server = get_build_server_info(InstallVars.get_install_var('BUILD_SERVER'))
bld_server = get_build_server_info(
RestoreVars.get_restore_var('BUILD_SERVER'))
LOG.info("Connecting to Build Server {} ....".format(
bld_server['name']))
bld_server_attr = dict()
bld_server_attr['name'] = bld_server['name']
bld_server_attr['server_ip'] = bld_server['ip']
bld_server_attr['prompt'] = r'{}@{}\:(.*)\$ '.format(
TestFileServer.get_user(), bld_server['name'])
bld_server_conn = install_helper.establish_ssh_connection(
bld_server_attr['name'],
user=TestFileServer.get_user(),
password=TestFileServer.get_password(),
initial_prompt=bld_server_attr['prompt'])
bld_server_conn.exec_cmd("bash")
bld_server_conn.set_prompt(bld_server_attr['prompt'])
bld_server_conn.deploy_ssh_key(install_helper.get_ssh_public_key())
bld_server_attr['ssh_conn'] = bld_server_conn
bld_server_obj = Server(**bld_server_attr)
# If controller is accessible, check if USB with backup files is avaialble
load_path = os.path.join(
BuildServerPath.DEFAULT_WORK_SPACE,
RestoreVars.get_restore_var("BACKUP_BUILDS_DIR"), backup_build_id)
InstallVars.set_install_var(tis_build_dir=load_path)
# set up feed for controller
LOG.fixture_step(
"Setting install feed in tuxlab for controller-0 ... ")
if 'vbox' not in lab['name'] and not RestoreVars.get_restore_var(
'skip_setup_feed'):
assert install_helper.set_network_boot_feed(bld_server_conn, load_path), \
"Fail to set up feed for controller"
if not RestoreVars.get_restore_var('skip_reinstall'):
# power off hosts
LOG.fixture_step("Powring off system hosts ... ")
install_helper.power_off_host(hostnames)
LOG.fixture_step("Booting controller-0 ... ")
is_cpe = (lab.get('system_type', 'Standard') == 'CPE')
low_latency = RestoreVars.get_restore_var('low_latency')
os.environ['XTERM'] = 'xterm'
install_helper.boot_controller(small_footprint=is_cpe,
system_restore=True,
low_latency=low_latency)
# establish ssh connection with controller
LOG.fixture_step(
"Establishing ssh connection with controller-0 after install..."
)
node_name_in_ini = r'{}.*\~\$ '.format(
install_helper.get_lab_info(controller_node.barcode)['name'])
controller_prompt = re.sub(r'([^\d])0*(\d+)', r'\1\2',
node_name_in_ini)
controller_prompt = controller_prompt + '|' + Prompt.TIS_NODE_PROMPT_BASE.format(
lab['name'].split('_')[0]) + '|' + Prompt.CONTROLLER_0
LOG.info('initial_prompt=' + controller_prompt)
controller_node.ssh_conn = install_helper.ssh_to_controller(
controller_node.host_ip, initial_prompt=controller_prompt)
LOG.info('Deploy ssh key')
controller_node.ssh_conn.deploy_ssh_key()
ControllerClient.set_active_controller(ssh_client=controller_node.ssh_conn)
con_ssh = controller_node.ssh_conn
tis_backup_files = pre_restore_checkup
backup_src = RestoreVars.get_restore_var('backup_src'.upper())
backup_src_path = RestoreVars.get_restore_var('backup_src_path'.upper())
if backup_src.lower() == 'local':
LOG.fixture_step(
"Transferring system backup file to controller-0 {} ... ".format(
HostLinuxUser.get_home()))
system_backup_file = [
file for file in tis_backup_files if "system.tgz" in file
].pop()
common.scp_from_test_server_to_active_controller(
"{}/{}".format(backup_src_path, system_backup_file),
HostLinuxUser.get_home())
assert con_ssh.exec_cmd("ls {}{}".format(HostLinuxUser.get_home(),
system_backup_file))[0] == 0, \
"Missing backup file {} in dir {}".format(system_backup_file, HostLinuxUser.get_home())
elif backup_src.lower() == 'usb':
tis_backup_files = pre_restore_checkup
usb_device_name = install_helper.get_usb_device_name(con_ssh=con_ssh)
usb_part_name = "{}2".format(usb_device_name)
assert usb_device_name, "No USB found "
LOG.fixture_step(
"USB flash drive found, checking for backup files ... ")
if len(tis_backup_files) == 0:
LOG.fixture_step("Checking for backup files in USB ... ")
usb_part_info = install_helper.get_usb_device_partition_info(
usb_device=usb_device_name, con_ssh=con_ssh)
assert usb_part_info and len(
usb_part_info) > 0, "No USB or partition found"
assert usb_part_name in usb_part_info.keys(
), "No {} partition exist in USB"
result, mount_point = install_helper.is_usb_mounted(
usb_device=usb_part_name)
if not result:
assert install_helper.mount_usb(usb_device=usb_part_name, con_ssh=con_ssh), \
"Unable to mount USB partition {}".format(usb_part_name)
tis_backup_files = install_helper.get_titanium_backup_filenames_usb(
usb_device=usb_part_name)
assert len(
tis_backup_files) >= 2, "Missing backup files: {}".format(
tis_backup_files)
else:
result, mount_point = install_helper.is_usb_mounted(
usb_device=usb_part_name)
if not result:
assert install_helper.mount_usb(usb_device=usb_part_name, con_ssh=con_ssh), \
"Unable to mount USB partition {}".format(usb_part_name)
_restore_setup = {
'lab': lab,
'output_dir': output_dir,
'build_server': bld_server_obj,
'tis_backup_files': tis_backup_files
}
return _restore_setup