def connect(hostname, port=10000, prefix=""): """ Connect to local domain socket and return the socket object. Arguments: - Requires the hostname of target, e.g. controller-0 - Requires TCP port if using Windows """ if prefix: prefix = "{}_".format(prefix) socketname = "/tmp/{}{}".format(prefix, hostname) if 'controller-0' in hostname: socketname += '_serial' LOG.info("Connecting to %s at %s", hostname, socketname) if platform == 'win32' or platform == 'win64': sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) else: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: if platform == 'win32' or platform == 'win64': sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.connect(('localhost', port)) else: sock.connect(socketname) except: LOG.info("Connection failed") pass # disconnect(sock) sock = None # TODO (WEI): double check this sock.setblocking(0) return sock
def vboxmanage_startvm(hostname=None, force=False): """ This allows you to power on a VM. """ assert hostname, "Hostname is required" if not force: LOG.info("Check if VM is running") running_vms = vboxmanage_list(option="runningvms") else: running_vms = [] if hostname.encode('utf-8') in running_vms: LOG.info("Host %s is already started", hostname) else: LOG.info("Powering on VM %s", hostname) result = subprocess.check_output(['vboxmanage', 'startvm', hostname], stderr=subprocess.STDOUT) LOG.info(result) # Wait for VM to start tmout = 20 while tmout: tmout -= 1 running_vms = vboxmanage_list(option="runningvms") if hostname.encode('utf-8') in running_vms: break time.sleep(1) else: raise "Failed to start VM: {}".format(hostname) LOG.info("VM '%s' started.", hostname)
def print_kpi(metric): if metric in STAGES: sec = METRICS[metric] LOG.info(" Time in stage '%s': %s ", metric, get_formated_time(sec)) elif metric == 'total' and start: duration = time.time() - start LOG.info(" Total time: %s", get_formated_time(duration))
def get_patches(cont0_stream, ctrlr0_ip=None, local_path=None, remote_host=None, release='R6', username='******', password='******'): """ Retrieves patches from remote_host or localhost if remote_host is None """ patch_dir = "/home/" + username + "/patches" serial.send_bytes(cont0_stream, "mkdir " + patch_dir) if local_path is None: local_path = env.FILEPATH + '{}/patches/'.format(release) remote_path = '/home/' + username + '/patches/' LOG.info("Remote host is {}".format(remote_host)) if remote_host is not None: if release == 'R6': pass elif release == 'R5': #patch_loc = env.Builds.R5['patches'] pass elif release == 'R4': patch_loc = env.Builds.R4['patches'] elif release == 'R3': patch_loc = env.Builds.R3['patches'] else: patch_loc = env.Builds.R2['patches'] for items in patch_loc: send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path, username=username, password=password) send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path, username=username, password=password) else: LOG.info("Retrieving patches from {}".format(local_path)) if not local_path.endswith('/') or not local_path.endswith('\\'): local_path = local_path + '/' send_dir(source=local_path, remote_host=ctrlr0_ip, destination=remote_path, username=username, password=password)
def connect(hostname, port=10000): """ Connect to local domain socket and return the socket object. Arguments: - Requires the hostname of target, e.g. controller-0 - Requires TCP port if using Windows """ # Need to power on host before we can connect vboxmanage.vboxmanage_startvm(hostname) if 'controller-0'in hostname: socketname = '/tmp/' + hostname + '_serial' else: socketname = "/tmp/{}".format(hostname) LOG.info("Connecting to {}".format(hostname)) if platform == 'win32' or platform == 'win64': sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) else: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: if platform == 'win32' or platform == 'win64': sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.connect(('localhost', port)) else: sock.connect(socketname) except: LOG.info("Connection failed") pass # disconnect(sock) sock = None # TODO (WEI): double check this sock.setblocking(0) return sock
def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'): """ This creates the required disks. """ assert hostname, "Hostname is required" assert disk_list, "A list of disk sizes is required" username = getpass.getuser() device_num = 0 port_num = 0 disk_count = 1 for disk in disk_list: if platform == 'win32' or platform == 'win64': ## TODO (WEI) fix it file_name = "C:\\Users\\" + username + "\\vbox_disks\\" + hostname + "_disk_{}".format(disk_count) else: file_name = vbox_home_dir + '/' + username + "/vbox_disks/" + hostname + "_disk_{}".format(disk_count) LOG.info("Creating disk {} with size {} on VM {} on device {} port {}".format(file_name, disk, hostname, device_num, port_num)) result = subprocess.check_output(['vboxmanage', 'createmedium', 'disk', '--size', str(disk), '--filename', file_name, '--format', 'vdi', '--variant', 'standard'], stderr=subprocess.STDOUT) LOG.info(result) vboxmanage_storageattach(hostname, "sata", "hdd", file_name + ".vdi", str(port_num), str(device_num)) disk_count += 1 port_num += 1
def install_host(stream, hostname, host_type, host_id): """ Initiates install of specified host. Requires controller-0 to be installed already. Args: stream(stream): Stream to cont0 hostname(str): Name of host host_type(str): Type of host being installed e.g. 'storage' or 'compute' host_id(int): id to identify host """ time.sleep(10) LOG.info("Installing %s with id %s", hostname, host_id) if host_type is 'controller': serial.send_bytes(stream, "system host-update {} personality=controller".format(host_id), expect_prompt=False) elif host_type is 'storage': serial.send_bytes(stream, "system host-update {} personality=storage".format(host_id), expect_prompt=False) else: serial.send_bytes(stream, "system host-update {} personality=compute hostname={}".format(host_id, hostname), expect_prompt=False) time.sleep(30)
def vboxmanage_deletemedium(hostname, vbox_home_dir='/home'): assert hostname, "Hostname is required" if platform == 'win32' or platform == 'win64': return username = getpass.getuser() vbox_home_dir = "{}/{}/vbox_disks/".format(vbox_home_dir, username) disk_list = [f for f in os.listdir(vbox_home_dir) if os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f] LOG.info("Disk mediums to delete: %s", disk_list) for disk in disk_list: LOG.info("Disconnecting disk %s from vbox.", disk) try: result = subprocess.check_output(['vboxmanage', 'closemedium', 'disk', "{}{}".format(vbox_home_dir, disk), '--delete'], stderr=subprocess.STDOUT) LOG.info(result) except subprocess.CalledProcessError as e: # Continue if failures, disk may not be present LOG.info("Error disconnecting disk, continuing. " "Details: stdout: %s stderr: %s", e.stdout, e.stderr) LOG.info("Removing backing file %s", disk) try: os.remove("{}{}".format(vbox_home_dir, disk)) except: pass
def disable_logout(stream): """ Disables automatic logout of users. Args: stream(stream): stream to cont0 """ LOG.info('Disabling automatic logout') serial.send_bytes(stream, "export TMOUT=0")
def vboxmanage_hostonlyifdelete(name="vboxnet0"): """ Deletes hostonly network. This is used as a work around for creating too many hostonlyifs. """ assert name, "Must provide network name" LOG.info("Removing Host-only Network") result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'remove', name], stderr=subprocess.STDOUT)
def vboxmanage_createvm(hostname=None): """ This creates a VM with the specified name. """ assert hostname, "Hostname is required" LOG.info("Creating VM {}".format(hostname)) result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register', '--ostype', 'Linux_64'], stderr=subprocess.STDOUT)
def vboxmanage_modifyvm(hostname=None, cpus=None, memory=None, nic=None, nictype=None, nicpromisc=None, nicnum=None, intnet=None, hostonlyadapter=None, natnetwork=None, uartbase=None, uartport=None, uartmode=None, uartpath=None, nicbootprio2=1, prefix=""): """ This modifies a VM with a specified name. """ assert hostname, "Hostname is required" # Add more semantic checks cmd = ['vboxmanage', 'modifyvm', hostname] if cpus: cmd.extend(['--cpus', cpus]) if memory: cmd.extend(['--memory', memory]) if nic and nictype and nicpromisc and nicnum: cmd.extend(['--nic{}'.format(nicnum), nic]) cmd.extend(['--nictype{}'.format(nicnum), nictype]) cmd.extend(['--nicpromisc{}'.format(nicnum), nicpromisc]) if intnet: if prefix: intnet = "{}-{}".format(prefix, intnet) else: intnet = "{}".format(intnet) cmd.extend(['--intnet{}'.format(nicnum), intnet]) if hostonlyadapter: cmd.extend(['--hostonlyadapter{}'.format(nicnum), hostonlyadapter]) if natnetwork: cmd.extend(['--nat-network{}'.format(nicnum), natnetwork]) elif nicnum and nictype == 'nat': cmd.extend(['--nic{}'.format(nicnum), 'nat']) if uartbase and uartport and uartmode and uartpath: cmd.extend(['--uart1']) cmd.extend(['{}'.format(uartbase)]) cmd.extend(['{}'.format(uartport)]) cmd.extend(['--uartmode1']) cmd.extend(['{}'.format(uartmode)]) if platform == 'win32' or platform == 'win64': cmd.extend(['{}'.format(env.PORT)]) env.PORT += 1 else: if prefix: prefix = "{}_".format(prefix) if 'controller-0' in hostname: cmd.extend(['{}{}{}_serial'.format(uartpath, prefix, hostname)]) else: cmd.extend(['{}{}{}'.format(uartpath, prefix, hostname)]) if nicbootprio2: cmd.extend(['--nicbootprio2']) cmd.extend(['{}'.format(nicbootprio2)]) cmd.extend(['--boot4']) cmd.extend(['net']) LOG.info(cmd) LOG.info("Updating VM %s configuration", hostname) result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def set_dns(stream, dns_ip): """ Perform DNS configuration on the system. """ LOG.info("Configuring DNS to %s.", dns_ip) serial.send_bytes(stream, "source /etc/nova/openrc; system dns-modify " "nameservers={}".format(dns_ip), prompt='keystone')
def reboot_host(stream, hostname): """ Reboots host specified Args: stream(): hostname(str): Host to reboot """ LOG.info("Rebooting %s", hostname) serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False) serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)
def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"): """ This creates a storage controller on the host. """ assert hostname, "Hostname is required" assert storectl, "Type of storage controller is required" LOG.info("Creating {} storage controller on VM {}".format(storectl, hostname)) result = subprocess.check_output(['vboxmanage', 'storagectl', hostname, '--name', storectl, '--add', storectl, '--hostiocache', hostiocache], stderr=subprocess.STDOUT)
def vboxmanage_restoresnapshot(host=None, name=None): """ This allows you to restore snapshot of a VM. """ assert host, "Hostname is required" assert name, "Need to provide the snapshot to restore" LOG.info("Restoring snapshot {} on VM {}".format(name, host)) result = subprocess.call(["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT) time.sleep(10)
def delete_lab(labname): node_list = get_all_vms(labname, option="vms") if vboxoptions.debug_rest: node_list.remove(labname + "-controller-0") if len(node_list) != 0: LOG.info("Deleting existing VMs: {}".format(node_list)) vboxmanage.vboxmanage_controlvms(node_list, "poweroff") time.sleep(5) vboxmanage.vboxmanage_deletevms(node_list)
def vboxmanage_createvm(hostname, labname): """ This creates a VM with the specified name. """ assert hostname, "Hostname is required" assert labname, "Labname is required" group = "/" + labname LOG.info("Creating VM %s", hostname) result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register', '--ostype', 'Linux_64', '--groups', group], stderr=subprocess.STDOUT)
def disconnect(sock): """ Disconnect a local doamin socket. Arguments: - Requires socket """ # Shutdown connection and release resources LOG.info("Disconnecting from socket") sock.shutdown(socket.SHUT_RDWR) sock.close()
def vboxmanage_takesnapshot(hosts=None, name=None): """ This allows you to take snapshot of VMs. """ assert hosts, "Hostname is required" assert name, "Need to provide a name for the snapshot" for host in hosts: LOG.info("Taking snapshot %s on VM %s", name, host) result = subprocess.call(["vboxmanage", "snapshot", host, "take", name], stderr=subprocess.STDOUT)
def vboxmanage_controlvms(hosts=None, action=None): """ This allows you to control a VM, e.g. pause, resume, etc. """ assert hosts, "Hostname is required" assert action, "Need to provide an action to execute" for host in hosts: LOG.info("Executing {} action on VM {}".format(action, host)) result = subprocess.call(["vboxmanage", "controlvm", host, action], stderr=subprocess.STDOUT) time.sleep(1)
def update_platform_cpus(stream, hostname, cpu_num=5): """ Update platform CPU allocation. """ LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname) serial.send_bytes( stream, "\nsource /etc/platform/openrc; system host-cpu-modify " "{} -f platform -p0 {}".format(hostname, cpu_num, prompt='keystone', timeout=300))
def get_all_vms(labname, option="vms"): node_list = [] vm_list = vboxmanage.vboxmanage_list(option) LOG.info("The following VMs are present on the system: {}".format(vm_list)) labname.encode('utf-8') for item in vm_list: if labname.encode('utf-8') in item and (b'controller-' in item or \ b'compute-' in item or b'storage-' in item): node_list.append(item.decode('utf-8')) #print ("node list ", node_list) return node_list
def sftp_send(source, remote_host, destination, username, password): """ Send files to remote server, usually controller-0 args: - source: full path to file including the filename e.g. /localhost/loadbuild/jenkins/CGCS_5.0_Host/latest_build/bootimage.iso - Remote host: name of host to log into, controller-0 by default e.g. yow-cgts4-lx.wrs.com - destination: where to store the file locally: /tmp/bootimage.iso """ LOG.info("Connecting to server {} with username {}".format(remote_host, username)) ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ## TODO(WEI): need to make this timeout handling better retry = 0 while retry < 8: try: ssh_client.connect(remote_host, username=username, password=password, look_for_keys=False, allow_agent=False) sftp_client = ssh_client.open_sftp() retry = 8 except Exception as e: LOG.info("******* try again") retry += 1 time.sleep(10) LOG.info("Sending file from {} to {}".format(source, destination)) sftp_client.put(source, destination) LOG.info("Done") sftp_client.close() ssh_client.close()
def sftp_send(source, remote_host, remote_port, destination, username, password): """ Send files to remote server """ LOG.info("Connecting to server %s with username %s", remote_host, username) ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ## TODO(WEI): need to make this timeout handling better retry = 0 while retry < 8: try: ssh_client.connect(remote_host, port=remote_port, username=username, password=password, look_for_keys=False, allow_agent=False) sftp_client = ssh_client.open_sftp() retry = 8 except Exception as e: LOG.info("******* try again") retry += 1 time.sleep(10) LOG.info("Sending file from %s to %s", source, destination) sftp_client.put(source, destination) LOG.info("Done") sftp_client.close() ssh_client.close()
def setup_networking(stream, release, ctrlr0_ip, hostadapter_ip, password='******'): """ Setup initial networking so we can transfer files. """ ip = ctrlr0_ip host_ip = hostadapter_ip if release == "R2": interface = "eth0" else: interface = "enp0s3" ret = serial.send_bytes(stream, "/sbin/ip address list", prompt=ctrlr0_ip, fail_ok=True, timeout=10) if ret != 0: LOG.info("Setting networking up.") else: LOG.info("Skipping networking setup") return LOG.info("{} being set up with ip {}".format(interface, ip)) serial.send_bytes(stream, "sudo /sbin/ip addr add {}/24 dev {}".format(ip, interface), expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) serial.send_bytes(stream, "sudo /sbin/ip link set {} up".format(interface), expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) serial.send_bytes(stream, "sudo route add default gw {}".format(host_ip), expect_prompt=False) host_helper.check_password(stream, password=password) NETWORKING_TIME = 60 LOG.info("Wait a minute for networking to be established") time.sleep(NETWORKING_TIME)
def take_snapshot(labname, snapshot_name, socks=None): vms = get_all_vms(labname, option="vms") runningvms = get_all_vms(labname, option="runningvms") LOG.info("#### Taking snapshot %s of lab %s", snapshot_name, labname) LOG.info("VMs in lab %s: %s", labname, vms) LOG.info("VMs running in lab %s: %s", labname, runningvms) hosts = len(vms) # Pause running VMs to take snapshot if len(runningvms) > 1: for node in runningvms: newpid = os.fork() if newpid == 0: vboxmanage_controlvms([node], "pause") os._exit(0) for node in vms: os.waitpid(0, 0) time.sleep(2) if hosts != 0: vboxmanage_takesnapshot(vms, snapshot_name) # Resume VMs after snapshot was taken if len(runningvms) > 1: for node in runningvms: newpid = os.fork() if newpid == 0: vboxmanage_controlvms([node], "resume") os._exit(0) for node in runningvms: os.waitpid(0, 0) time.sleep(10) # Wait for VM serial port to stabilize, otherwise it may refuse to connect if runningvms: new_vms = get_all_vms(labname, option="runningvms") retry = 0 while retry < 20: LOG.info("Waiting for VMs to come up running after taking snapshot..." "Up VMs are %s ", new_vms) if len(runningvms) < len(new_vms): time.sleep(1) new_vms = get_all_vms(labname, option="runningvms") retry += 1 else: LOG.info("All VMs %s are up running after taking snapshot...", vms) break
def vboxmanage_storageattach(hostname=None, storectl="sata", storetype="hdd", disk=None, port_num="0", device_num="0"): """ This attaches a disk to a controller. """ assert hostname, "Hostname is required" assert disk, "Disk name is required" assert storectl, "Name of storage controller is required" assert storetype, "Type of storage controller is required" LOG.info("Attaching {} storage to storage controller {} on VM {}".format(storetype, storectl, hostname)) result = subprocess.check_output(['vboxmanage', 'storageattach', hostname, '--storagectl', storectl, '--medium', disk, '--type', storetype, '--port', port_num, '--device', device_num], stderr=subprocess.STDOUT) return result
def send_dir(source, remote_host, remote_port, destination, username, password, follow_links=True, clear_known_hosts=True): # Only works from linux for now if not source.endswith('/') or not source.endswith('\\'): source = source + '/' params = { 'source': source, 'remote_host': remote_host, 'destination': destination, 'port': remote_port, 'username': username, 'password': password, 'follow_links': "L" if follow_links else "", } if clear_known_hosts: if remote_host == '127.0.0.1': keygen_arg = "[127.0.0.1]:{}".format(remote_port) else: keygen_arg = remote_host cmd = 'ssh-keygen -f "/home/%s/.ssh/known_hosts" -R' \ ' %s', getpass.getuser(), keygen_arg LOG.info("CMD: %s", cmd) process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) for line in iter(process.stdout.readline, b''): LOG.info("%s", line.decode("utf-8").strip()) process.wait() LOG.info("Running rsync of dir: {source} ->" \ "{username}@{remote_host}:{destination}".format(**params)) cmd = ( "rsync -av{follow_links} " "--rsh=\"/usr/bin/sshpass -p {password} ssh -p {port} -o StrictHostKeyChecking=no -l {username}\" " "{source}* {username}@{remote_host}:{destination}".format(**params)) LOG.info("CMD: %s", cmd) process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) for line in iter(process.stdout.readline, b''): LOG.info("%s", line.decode("utf-8").strip()) process.wait() if process.returncode: raise Exception("Error in rsync, return code:{}".format( process.returncode))
def vboxmanage_hostonlyifcreate(name="vboxnet0", ip=None, netmask=None): """ This creates a hostonly network for systems to communicate. """ assert name, "Must provide network name" assert ip, "Must provide an OAM IP" assert netmask, "Must provide an OAM Netmask" LOG.info("Creating Host-only Network") result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'create'], stderr=subprocess.STDOUT) LOG.info("Provisioning {} with IP {} and Netmask {}".format(name, ip, netmask)) result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'ipconfig', name, '--ip', ip, '--netmask', netmask], stderr=subprocess.STDOUT)