def connect(hostname, port=10000, prefix=""):
"""
Connect to local domain socket and return the socket object.
Arguments:
- Requires the hostname of target, e.g. controller-0
- Requires TCP port if using Windows
"""
if prefix:
prefix = "{}_".format(prefix)
socketname = "/tmp/{}{}".format(prefix, hostname)
if 'controller-0' in hostname:
socketname += '_serial'
LOG.info("Connecting to %s at %s", hostname, socketname)
if platform == 'win32' or platform == 'win64':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
else:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
if platform == 'win32' or platform == 'win64':
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.connect(('localhost', port))
else:
sock.connect(socketname)
except:
LOG.info("Connection failed")
pass
# disconnect(sock)
sock = None
# TODO (WEI): double check this
sock.setblocking(0)
return sock
def vboxmanage_startvm(hostname=None, force=False):
"""
This allows you to power on a VM.
"""
assert hostname, "Hostname is required"
if not force:
LOG.info("Check if VM is running")
running_vms = vboxmanage_list(option="runningvms")
else:
running_vms = []
if hostname.encode('utf-8') in running_vms:
LOG.info("Host %s is already started", hostname)
else:
LOG.info("Powering on VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'startvm',
hostname], stderr=subprocess.STDOUT)
LOG.info(result)
# Wait for VM to start
tmout = 20
while tmout:
tmout -= 1
running_vms = vboxmanage_list(option="runningvms")
if hostname.encode('utf-8') in running_vms:
break
time.sleep(1)
else:
raise "Failed to start VM: {}".format(hostname)
LOG.info("VM '%s' started.", hostname)
def print_kpi(metric):
if metric in STAGES:
sec = METRICS[metric]
LOG.info(" Time in stage '%s': %s ", metric, get_formated_time(sec))
elif metric == 'total' and start:
duration = time.time() - start
LOG.info(" Total time: %s", get_formated_time(duration))
def get_patches(cont0_stream, ctrlr0_ip=None, local_path=None, remote_host=None, release='R6', username='******', password='******'):
"""
Retrieves patches from remote_host or localhost if remote_host is None
"""
patch_dir = "/home/" + username + "/patches"
serial.send_bytes(cont0_stream, "mkdir " + patch_dir)
if local_path is None:
local_path = env.FILEPATH + '{}/patches/'.format(release)
remote_path = '/home/' + username + '/patches/'
LOG.info("Remote host is {}".format(remote_host))
if remote_host is not None:
if release == 'R6':
pass
elif release == 'R5':
#patch_loc = env.Builds.R5['patches']
pass
elif release == 'R4':
patch_loc = env.Builds.R4['patches']
elif release == 'R3':
patch_loc = env.Builds.R3['patches']
else:
patch_loc = env.Builds.R2['patches']
for items in patch_loc:
send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path,
username=username, password=password)
send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path,
username=username, password=password)
else:
LOG.info("Retrieving patches from {}".format(local_path))
if not local_path.endswith('/') or not local_path.endswith('\\'):
local_path = local_path + '/'
send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path,
username=username, password=password)
def connect(hostname, port=10000):
"""
Connect to local domain socket and return the socket object.
Arguments:
- Requires the hostname of target, e.g. controller-0
- Requires TCP port if using Windows
"""
# Need to power on host before we can connect
vboxmanage.vboxmanage_startvm(hostname)
if 'controller-0'in hostname:
socketname = '/tmp/' + hostname + '_serial'
else:
socketname = "/tmp/{}".format(hostname)
LOG.info("Connecting to {}".format(hostname))
if platform == 'win32' or platform == 'win64':
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
else:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
if platform == 'win32' or platform == 'win64':
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.connect(('localhost', port))
else:
sock.connect(socketname)
except:
LOG.info("Connection failed")
pass
# disconnect(sock)
sock = None
# TODO (WEI): double check this
sock.setblocking(0)
return sock
def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'):
"""
This creates the required disks.
"""
assert hostname, "Hostname is required"
assert disk_list, "A list of disk sizes is required"
username = getpass.getuser()
device_num = 0
port_num = 0
disk_count = 1
for disk in disk_list:
if platform == 'win32' or platform == 'win64':
## TODO (WEI) fix it
file_name = "C:\\Users\\" + username + "\\vbox_disks\\" + hostname + "_disk_{}".format(disk_count)
else:
file_name = vbox_home_dir + '/' + username + "/vbox_disks/" + hostname + "_disk_{}".format(disk_count)
LOG.info("Creating disk {} with size {} on VM {} on device {} port {}".format(file_name, disk, hostname, device_num, port_num))
result = subprocess.check_output(['vboxmanage', 'createmedium', 'disk', '--size', str(disk), '--filename',
file_name, '--format', 'vdi', '--variant', 'standard'],
stderr=subprocess.STDOUT)
LOG.info(result)
vboxmanage_storageattach(hostname, "sata", "hdd", file_name + ".vdi", str(port_num), str(device_num))
disk_count += 1
port_num += 1
def install_host(stream, hostname, host_type, host_id):
"""
Initiates install of specified host. Requires controller-0 to be installed already.
Args:
stream(stream): Stream to cont0
hostname(str): Name of host
host_type(str): Type of host being installed e.g. 'storage' or 'compute'
host_id(int): id to identify host
"""
time.sleep(10)
LOG.info("Installing %s with id %s", hostname, host_id)
if host_type is 'controller':
serial.send_bytes(stream,
"system host-update {} personality=controller".format(host_id),
expect_prompt=False)
elif host_type is 'storage':
serial.send_bytes(stream,
"system host-update {} personality=storage".format(host_id),
expect_prompt=False)
else:
serial.send_bytes(stream,
"system host-update {} personality=compute hostname={}".format(host_id,
hostname),
expect_prompt=False)
time.sleep(30)
def vboxmanage_deletemedium(hostname, vbox_home_dir='/home'):
assert hostname, "Hostname is required"
if platform == 'win32' or platform == 'win64':
return
username = getpass.getuser()
vbox_home_dir = "{}/{}/vbox_disks/".format(vbox_home_dir, username)
disk_list = [f for f in os.listdir(vbox_home_dir) if
os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f]
LOG.info("Disk mediums to delete: %s", disk_list)
for disk in disk_list:
LOG.info("Disconnecting disk %s from vbox.", disk)
try:
result = subprocess.check_output(['vboxmanage', 'closemedium', 'disk',
"{}{}".format(vbox_home_dir, disk), '--delete'],
stderr=subprocess.STDOUT)
LOG.info(result)
except subprocess.CalledProcessError as e:
# Continue if failures, disk may not be present
LOG.info("Error disconnecting disk, continuing. "
"Details: stdout: %s stderr: %s", e.stdout, e.stderr)
LOG.info("Removing backing file %s", disk)
try:
os.remove("{}{}".format(vbox_home_dir, disk))
except:
pass
def disable_logout(stream):
"""
Disables automatic logout of users.
Args:
stream(stream): stream to cont0
"""
LOG.info('Disabling automatic logout')
serial.send_bytes(stream, "export TMOUT=0")
def vboxmanage_hostonlyifdelete(name="vboxnet0"):
"""
Deletes hostonly network. This is used as a work around for creating too many hostonlyifs.
"""
assert name, "Must provide network name"
LOG.info("Removing Host-only Network")
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'remove', name], stderr=subprocess.STDOUT)
def vboxmanage_createvm(hostname=None):
"""
This creates a VM with the specified name.
"""
assert hostname, "Hostname is required"
LOG.info("Creating VM {}".format(hostname))
result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register', '--ostype',
'Linux_64'], stderr=subprocess.STDOUT)
def vboxmanage_modifyvm(hostname=None, cpus=None, memory=None, nic=None,
nictype=None, nicpromisc=None, nicnum=None,
intnet=None, hostonlyadapter=None,
natnetwork=None, uartbase=None, uartport=None,
uartmode=None, uartpath=None, nicbootprio2=1, prefix=""):
"""
This modifies a VM with a specified name.
"""
assert hostname, "Hostname is required"
# Add more semantic checks
cmd = ['vboxmanage', 'modifyvm', hostname]
if cpus:
cmd.extend(['--cpus', cpus])
if memory:
cmd.extend(['--memory', memory])
if nic and nictype and nicpromisc and nicnum:
cmd.extend(['--nic{}'.format(nicnum), nic])
cmd.extend(['--nictype{}'.format(nicnum), nictype])
cmd.extend(['--nicpromisc{}'.format(nicnum), nicpromisc])
if intnet:
if prefix:
intnet = "{}-{}".format(prefix, intnet)
else:
intnet = "{}".format(intnet)
cmd.extend(['--intnet{}'.format(nicnum), intnet])
if hostonlyadapter:
cmd.extend(['--hostonlyadapter{}'.format(nicnum), hostonlyadapter])
if natnetwork:
cmd.extend(['--nat-network{}'.format(nicnum), natnetwork])
elif nicnum and nictype == 'nat':
cmd.extend(['--nic{}'.format(nicnum), 'nat'])
if uartbase and uartport and uartmode and uartpath:
cmd.extend(['--uart1'])
cmd.extend(['{}'.format(uartbase)])
cmd.extend(['{}'.format(uartport)])
cmd.extend(['--uartmode1'])
cmd.extend(['{}'.format(uartmode)])
if platform == 'win32' or platform == 'win64':
cmd.extend(['{}'.format(env.PORT)])
env.PORT += 1
else:
if prefix:
prefix = "{}_".format(prefix)
if 'controller-0' in hostname:
cmd.extend(['{}{}{}_serial'.format(uartpath, prefix, hostname)])
else:
cmd.extend(['{}{}{}'.format(uartpath, prefix, hostname)])
if nicbootprio2:
cmd.extend(['--nicbootprio2'])
cmd.extend(['{}'.format(nicbootprio2)])
cmd.extend(['--boot4'])
cmd.extend(['net'])
LOG.info(cmd)
LOG.info("Updating VM %s configuration", hostname)
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def set_dns(stream, dns_ip):
"""
Perform DNS configuration on the system.
"""
LOG.info("Configuring DNS to %s.", dns_ip)
serial.send_bytes(stream,
"source /etc/nova/openrc; system dns-modify "
"nameservers={}".format(dns_ip),
prompt='keystone')
def reboot_host(stream, hostname):
"""
Reboots host specified
Args:
stream():
hostname(str): Host to reboot
"""
LOG.info("Rebooting %s", hostname)
serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False)
serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)
def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):
"""
This creates a storage controller on the host.
"""
assert hostname, "Hostname is required"
assert storectl, "Type of storage controller is required"
LOG.info("Creating {} storage controller on VM {}".format(storectl, hostname))
result = subprocess.check_output(['vboxmanage', 'storagectl', hostname, '--name', storectl, '--add', storectl, '--hostiocache', hostiocache],
stderr=subprocess.STDOUT)
def vboxmanage_restoresnapshot(host=None, name=None):
"""
This allows you to restore snapshot of a VM.
"""
assert host, "Hostname is required"
assert name, "Need to provide the snapshot to restore"
LOG.info("Restoring snapshot {} on VM {}".format(name, host))
result = subprocess.call(["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT)
time.sleep(10)
def delete_lab(labname):
node_list = get_all_vms(labname, option="vms")
if vboxoptions.debug_rest:
node_list.remove(labname + "-controller-0")
if len(node_list) != 0:
LOG.info("Deleting existing VMs: {}".format(node_list))
vboxmanage.vboxmanage_controlvms(node_list, "poweroff")
time.sleep(5)
vboxmanage.vboxmanage_deletevms(node_list)
def vboxmanage_createvm(hostname, labname):
"""
This creates a VM with the specified name.
"""
assert hostname, "Hostname is required"
assert labname, "Labname is required"
group = "/" + labname
LOG.info("Creating VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register',
'--ostype', 'Linux_64', '--groups', group],
stderr=subprocess.STDOUT)
def disconnect(sock):
"""
Disconnect a local doamin socket.
Arguments:
- Requires socket
"""
# Shutdown connection and release resources
LOG.info("Disconnecting from socket")
sock.shutdown(socket.SHUT_RDWR)
sock.close()
def vboxmanage_takesnapshot(hosts=None, name=None):
"""
This allows you to take snapshot of VMs.
"""
assert hosts, "Hostname is required"
assert name, "Need to provide a name for the snapshot"
for host in hosts:
LOG.info("Taking snapshot %s on VM %s", name, host)
result = subprocess.call(["vboxmanage", "snapshot", host, "take",
name], stderr=subprocess.STDOUT)
def vboxmanage_controlvms(hosts=None, action=None):
"""
This allows you to control a VM, e.g. pause, resume, etc.
"""
assert hosts, "Hostname is required"
assert action, "Need to provide an action to execute"
for host in hosts:
LOG.info("Executing {} action on VM {}".format(action, host))
result = subprocess.call(["vboxmanage", "controlvm", host, action], stderr=subprocess.STDOUT)
time.sleep(1)
def update_platform_cpus(stream, hostname, cpu_num=5):
"""
Update platform CPU allocation.
"""
LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num,
hostname)
serial.send_bytes(
stream, "\nsource /etc/platform/openrc; system host-cpu-modify "
"{} -f platform -p0 {}".format(hostname,
cpu_num,
prompt='keystone',
timeout=300))
def get_all_vms(labname, option="vms"):
node_list = []
vm_list = vboxmanage.vboxmanage_list(option)
LOG.info("The following VMs are present on the system: {}".format(vm_list))
labname.encode('utf-8')
for item in vm_list:
if labname.encode('utf-8') in item and (b'controller-' in item or \
b'compute-' in item or b'storage-' in item):
node_list.append(item.decode('utf-8'))
#print ("node list ", node_list)
return node_list
def sftp_send(source, remote_host, destination, username, password):
"""
Send files to remote server, usually controller-0
args:
- source: full path to file including the filename
e.g. /localhost/loadbuild/jenkins/CGCS_5.0_Host/latest_build/bootimage.iso
- Remote host: name of host to log into, controller-0 by default
e.g. yow-cgts4-lx.wrs.com
- destination: where to store the file locally: /tmp/bootimage.iso
"""
LOG.info("Connecting to server {} with username {}".format(remote_host, username))
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
## TODO(WEI): need to make this timeout handling better
retry = 0
while retry < 8:
try:
ssh_client.connect(remote_host, username=username, password=password, look_for_keys=False, allow_agent=False)
sftp_client = ssh_client.open_sftp()
retry = 8
except Exception as e:
LOG.info("******* try again")
retry += 1
time.sleep(10)
LOG.info("Sending file from {} to {}".format(source, destination))
sftp_client.put(source, destination)
LOG.info("Done")
sftp_client.close()
ssh_client.close()
def sftp_send(source, remote_host, remote_port, destination, username,
password):
"""
Send files to remote server
"""
LOG.info("Connecting to server %s with username %s", remote_host, username)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
## TODO(WEI): need to make this timeout handling better
retry = 0
while retry < 8:
try:
ssh_client.connect(remote_host,
port=remote_port,
username=username,
password=password,
look_for_keys=False,
allow_agent=False)
sftp_client = ssh_client.open_sftp()
retry = 8
except Exception as e:
LOG.info("******* try again")
retry += 1
time.sleep(10)
LOG.info("Sending file from %s to %s", source, destination)
sftp_client.put(source, destination)
LOG.info("Done")
sftp_client.close()
ssh_client.close()
def setup_networking(stream, release, ctrlr0_ip, hostadapter_ip, password='******'):
"""
Setup initial networking so we can transfer files.
"""
ip = ctrlr0_ip
host_ip = hostadapter_ip
if release == "R2":
interface = "eth0"
else:
interface = "enp0s3"
ret = serial.send_bytes(stream, "/sbin/ip address list", prompt=ctrlr0_ip, fail_ok=True, timeout=10)
if ret != 0:
LOG.info("Setting networking up.")
else:
LOG.info("Skipping networking setup")
return
LOG.info("{} being set up with ip {}".format(interface, ip))
serial.send_bytes(stream, "sudo /sbin/ip addr add {}/24 dev {}".format(ip, interface), expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
serial.send_bytes(stream, "sudo /sbin/ip link set {} up".format(interface), expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
serial.send_bytes(stream, "sudo route add default gw {}".format(host_ip), expect_prompt=False)
host_helper.check_password(stream, password=password)
NETWORKING_TIME = 60
LOG.info("Wait a minute for networking to be established")
time.sleep(NETWORKING_TIME)
def take_snapshot(labname, snapshot_name, socks=None):
vms = get_all_vms(labname, option="vms")
runningvms = get_all_vms(labname, option="runningvms")
LOG.info("#### Taking snapshot %s of lab %s", snapshot_name, labname)
LOG.info("VMs in lab %s: %s", labname, vms)
LOG.info("VMs running in lab %s: %s", labname, runningvms)
hosts = len(vms)
# Pause running VMs to take snapshot
if len(runningvms) > 1:
for node in runningvms:
newpid = os.fork()
if newpid == 0:
vboxmanage_controlvms([node], "pause")
os._exit(0)
for node in vms:
os.waitpid(0, 0)
time.sleep(2)
if hosts != 0:
vboxmanage_takesnapshot(vms, snapshot_name)
# Resume VMs after snapshot was taken
if len(runningvms) > 1:
for node in runningvms:
newpid = os.fork()
if newpid == 0:
vboxmanage_controlvms([node], "resume")
os._exit(0)
for node in runningvms:
os.waitpid(0, 0)
time.sleep(10) # Wait for VM serial port to stabilize, otherwise it may refuse to connect
if runningvms:
new_vms = get_all_vms(labname, option="runningvms")
retry = 0
while retry < 20:
LOG.info("Waiting for VMs to come up running after taking snapshot..."
"Up VMs are %s ", new_vms)
if len(runningvms) < len(new_vms):
time.sleep(1)
new_vms = get_all_vms(labname, option="runningvms")
retry += 1
else:
LOG.info("All VMs %s are up running after taking snapshot...", vms)
break
def vboxmanage_storageattach(hostname=None, storectl="sata", storetype="hdd", disk=None, port_num="0",
device_num="0"):
"""
This attaches a disk to a controller.
"""
assert hostname, "Hostname is required"
assert disk, "Disk name is required"
assert storectl, "Name of storage controller is required"
assert storetype, "Type of storage controller is required"
LOG.info("Attaching {} storage to storage controller {} on VM {}".format(storetype, storectl, hostname))
result = subprocess.check_output(['vboxmanage', 'storageattach', hostname, '--storagectl', storectl,
'--medium', disk, '--type', storetype, '--port', port_num, '--device',
device_num], stderr=subprocess.STDOUT)
return result
def send_dir(source,
remote_host,
remote_port,
destination,
username,
password,
follow_links=True,
clear_known_hosts=True):
# Only works from linux for now
if not source.endswith('/') or not source.endswith('\\'):
source = source + '/'
params = {
'source': source,
'remote_host': remote_host,
'destination': destination,
'port': remote_port,
'username': username,
'password': password,
'follow_links': "L" if follow_links else "",
}
if clear_known_hosts:
if remote_host == '127.0.0.1':
keygen_arg = "[127.0.0.1]:{}".format(remote_port)
else:
keygen_arg = remote_host
cmd = 'ssh-keygen -f "/home/%s/.ssh/known_hosts" -R' \
' %s', getpass.getuser(), keygen_arg
LOG.info("CMD: %s", cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
LOG.info("Running rsync of dir: {source} ->" \
"{username}@{remote_host}:{destination}".format(**params))
cmd = (
"rsync -av{follow_links} "
"--rsh=\"/usr/bin/sshpass -p {password} ssh -p {port} -o StrictHostKeyChecking=no -l {username}\" "
"{source}* {username}@{remote_host}:{destination}".format(**params))
LOG.info("CMD: %s", cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
if process.returncode:
raise Exception("Error in rsync, return code:{}".format(
process.returncode))
def vboxmanage_hostonlyifcreate(name="vboxnet0", ip=None, netmask=None):
"""
This creates a hostonly network for systems to communicate.
"""
assert name, "Must provide network name"
assert ip, "Must provide an OAM IP"
assert netmask, "Must provide an OAM Netmask"
LOG.info("Creating Host-only Network")
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'create'], stderr=subprocess.STDOUT)
LOG.info("Provisioning {} with IP {} and Netmask {}".format(name, ip, netmask))
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'ipconfig', name, '--ip', ip, '--netmask',
netmask], stderr=subprocess.STDOUT)