def get_domains():
"""
Return an array of Domain objects
Query terraform to get the names of the domains to create the objects.
"""
domains = []
cmd = "terraform output -json domain_names"
[ret, output] = libutils.execute_bash_cmd(cmd)
domain_names = json.loads(output)['value']
cmd = "terraform output -json domain_ips"
[ret, output] = libutils.execute_bash_cmd(cmd)
domain_ips = json.loads(output)['value']
# format of domain_names: ['name1', 'name2']
# format of domain_ips: [['10.40.1.81'], ['10.40.1.221']] or [[],[]]
i = 0
while i < len(domain_names):
if (domain_ips[i] == []):
ip = None
else:
ip = domain_ips[i][0]
domains.append(Domain(domain_names[i], ip))
i += 1
return domains
def execute_cmd(self, cmd, timeout=300, exit_on_failure=True):
self.logger.info("execute_cmd '{}'".format(cmd))
str = qau.generate_guest_exec_str(self.name, cmd)
out_json = libutils.execute_bash_cmd(str)[1]
pid = qau.get_pid(out_json)
self.logger.debug("The command has PID={}".format(pid))
i = 0
str = qau.generate_guest_exec_status(self.name, pid)
while i < timeout:
out_json = libutils.execute_bash_cmd(str)[1]
if qau.process_is_exited(out_json):
retcode = qau.get_ret_code(out_json)
output = qau.get_output(out_json)
if (retcode != 0):
err_data = qau.get_output(out_json, 'err-data')
self.logger.error("The command failed with exit code {}. "
"Reason:\n {}".format(
retcode, err_data))
if (exit_on_failure):
raise libutils.TrfmCommandFailed()
else:
self.logger.debug("The command '{}' on the domain '{}' "
"succedded.\nOUTPUT:\n{}".format(
cmd, self.name, output))
if (retcode != 0 and exit_on_failure):
raise libutils.TrfmCommandFailed()
return [retcode, output]
time.sleep(1)
i += 1
self.logger.error(
"The command '{}' on the domain '{}' timed out.".format(
cmd, self.name))
if (exit_on_failure):
raise libutils.TrfmCommandTimeout()
def deploy(self):
""" Deploy Environment
It creates the Terraform environment from the given .tf file
"""
self.logger.info("Deploying Terraform Environment ...")
try:
cmd = 'terraform init'
if ('LOG_COLORS' not in os.environ):
cmd = ("{} -no-color".format(cmd))
libutils.execute_bash_cmd(cmd, cwd=self.workdir)
except (libutils.TrfmCommandFailed, libutils.TrfmCommandTimeout) as e:
self.logger.error(e)
sys.exit(-1)
try:
cmd = "terraform apply -input=false -auto-approve {}".format(
self.tf_vars)
if ('LOG_COLORS' not in os.environ):
cmd = ("{} -no-color".format(cmd))
libutils.execute_bash_cmd(cmd, timeout=1000, cwd=self.workdir)
except (libutils.TrfmCommandFailed, libutils.TrfmCommandTimeout) as e:
self.logger.error(e)
self.clean()
sys.exit(-1)
def check_qemu_agent(self):
str = qau.generate_guest_ping_str(self.name)
try:
libutils.execute_bash_cmd(str)
return True
except libutils.TrfmCommandFailed:
return False
def deploy(self):
""" Deploy Environment
It creates the Terraform environment from the given .tf file
If snapshots is set to True, after the domains are up, it will create a
snapshot for each domain in case they are needed to be reverted
at a certain point of the test flow.
"""
self.logger.info("Deploying Terraform Environment ...")
try:
cmd = 'terraform init'
if ('LOG_COLORS' not in os.environ):
cmd = ("{} -no-color".format(cmd))
[ret, output] = libutils.execute_bash_cmd(cmd)
except (libutils.TrfmCommandFailed, libutils.TrfmCommandTimeout) as e:
self.logger.error(e)
self.clean(remove_terraform_env=False)
sys.exit(-1)
try:
cmd = ("terraform apply -auto-approve "
"-var \"basename={}\" "
"-var \"image={}\" "
"-var \"network={}\" "
"-var \"cores={}\" "
"-var \"ram={}\" "
"-var \"count={}\"".format(self.basename, self.image,
self.networks[0], self.cores,
self.ram, self.num_domains))
if ('LOG_COLORS' not in os.environ):
cmd = ("{} -no-color".format(cmd))
[ret, output] = libutils.execute_bash_cmd(cmd, timeout=400)
except (libutils.TrfmCommandFailed, libutils.TrfmCommandTimeout) as e:
self.logger.error(e)
self.clean()
sys.exit(-1)
self.domains = self.get_domains()
self.logger.info("Waiting for domains to be ready...")
for domain in self.domains:
domain.wait_for_qemu_agent_ready()
if (domain.ip is not None):
domain.wait_for_ip_ready()
domain.wait_for_ssh_ready()
if (self.snapshots):
self.logger.debug("Creating snapshots of domains...")
for domain in self.domains:
try:
domain.snapshot(action='create')
except libutils.TrfmSnapshotFailed:
sys.exit(-1)
self.logger.success("Environment deployed successfully.")
def execute_cmd(self, cmd, timeout=300, exit_on_failure=True):
"""
Execute a command.
Executes a command through qemu-agent-command, which is a daemon
program running inside the domain. It uses 'guest-exec' to run a
command, querys the result 'guest-exec-status' until the command ends
and returns a exit code and potential stdout or stderr.
If the command doesn't finish before a certain 'timeout', the method
with raise an exception if 'exit_on_failure' is set to True.
For more info about qemu agent, refer to:
https://wiki.libvirt.org/page/Qemu_guest_agent
"""
if not self.check_qemu_agent():
raise libutils.TrfmQemuAgentNotReady("Qemu-agent is not running "
"on the domain")
cmd = cmd.replace('"', '\\"').replace('\n', '\\n')
self.logger.debug("execute_cmd '{}'".format(cmd))
str = qau.generate_guest_exec_str(self.name, cmd)
out_json = libutils.execute_bash_cmd(str)
pid = qau.get_pid(out_json)
self.logger.debug("The command has PID={}".format(pid))
i = 0
str = qau.generate_guest_exec_status(self.name, pid)
while i < timeout:
out_json = libutils.execute_bash_cmd(str)
if qau.process_is_exited(out_json):
retcode = qau.get_ret_code(out_json)
output = qau.get_output(out_json)
if (retcode != 0):
err_data = qau.get_output(out_json, 'err-data')
self._print_log(cmd, retcode, err_data)
if (exit_on_failure):
raise libutils.TrfmCommandFailed
else:
self._print_log(cmd, retcode, output)
if (retcode != 0 and exit_on_failure):
raise libutils.TrfmCommandFailed
return [retcode, output]
time.sleep(1)
i += 1
self.logger.error(
"The command '{}' on the domain '{}' timed out.".format(
cmd, self.name))
if (exit_on_failure):
raise libutils.TrfmCommandTimeout
def get_network_octet():
"""
Find a non-used network in the system
To allow multiple environments co-exist, network ranges can't be
hardcoded. Otherwise, new libvirt virtual networks can't be created.
The default environment will create a network with range 10.X.0.0/24,
where X will be calculated dynamically according to the existing
networks on the system, starting from X=0, this offers 255 possible
isolated environments running at the same time.
"""
global file_lock
locks_dir = '/tmp/qatrfm'
Path(locks_dir).mkdir(exist_ok=True)
x = 0
while x < 255:
try:
file_lock = open('{}/{}'.format(locks_dir, x), 'a')
output = libutils.execute_bash_cmd(
'ip a|grep 10.{}||true'.format(x))
if output == '':
fcntl.flock(file_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return x
except IOError:
pass
x += 1
raise Exception("Cannot find available network range")
def clean(self):
""" Destroys the Terraform environment """
self.logger.info("Removing Terraform Environment...")
cmd = "terraform destroy -input=false -auto-approve {}".format(
self.tf_vars)
if ('LOG_COLORS' not in os.environ):
cmd = ("{} -no-color".format(cmd))
try:
libutils.execute_bash_cmd(cmd, cwd=self.workdir)
except (libutils.TrfmCommandFailed, libutils.TrfmCommandTimeout) as e:
self.logger.error(e)
shutil.rmtree(self.workdir)
raise (e)
shutil.rmtree(self.workdir)
self.logger.success("Environment clean")
def clean(self, remove_terraform_env=True):
""" Destroys the Terraform environment """
self.logger.info("Removing Terraform Environment...")
if (remove_terraform_env):
if (self.snapshots):
for domain in self.domains:
try:
domain.snapshot(action='delete')
except libutils.TrfmSnapshotFailed as e:
shutil.rmtree(self.workdir)
raise (e)
cmd = ("terraform destroy -auto-approve "
"-var \"basename={}\" "
"-var \"image={}\" "
"-var \"network={}\" "
"-var \"cores={}\" "
"-var \"ram={}\" "
"-var \"count={}\"".format(self.basename, self.image,
self.networks[0], self.cores,
self.ram, self.num_domains))
if ('LOG_COLORS' not in os.environ):
cmd = ("{} -no-color".format(cmd))
try:
[ret, output] = libutils.execute_bash_cmd(cmd)
except (libutils.TrfmCommandFailed,
libutils.TrfmCommandTimeout) as e:
self.logger.error(e)
shutil.rmtree(self.workdir)
raise (e)
shutil.rmtree(self.workdir)
self.logger.success("Environment clean")
def wait_for_ssh_ready(self, timeout=300):
"""
Waits for domain's TCP port 22 (SSH) is reachable.
The user is responsible to use an image which allows ingress traffic in
port 22 TCP. Any firewall rules must be disabled beforehand.
"""
i = 0
while (i < int(timeout / 10)):
try:
cmd = "nc -vz -w 1 {} 22".format(self.ip)
libutils.execute_bash_cmd(cmd)
self.logger.debug("SSH on port 22 reachable")
return
except libutils.TrfmCommandFailed:
i += 1
time.sleep(10)
self.logger.warning("SSH is not available on the domain.")
def wait_for_ip_ready(self, timeout=300):
"""
Waits until domain's ip is pingable.
The user is responsible to use an image which allows ingress ICMP
traffic.
"""
i = 0
while (i < int(timeout / 10)):
try:
cmd = "ping -c 1 {}".format(self.ip)
libutils.execute_bash_cmd(cmd)
self.logger.debug("IP '{}' reachable".format(self.ip))
return
except libutils.TrfmCommandFailed:
i += 1
time.sleep(10)
raise libutils.TrfmDomainTimeout
def clean(self):
self.logger.info("Remove Environment")
if (self.deployed):
[ret, output
] = (libutils.execute_bash_cmd('terraform destroy -auto-approve'))
if ret != 0:
self.logger.error("Cannot clean environment")
else:
self.logger.debug("Terraform destroyed")
shutil.rmtree(self.workdir)
self.logger.info("Environment cleaned")
def get_domains():
""" Return an array of Domain objects """
domains = []
cmd = "terraform output -json vm_names"
[ret, output] = libutils.execute_bash_cmd(cmd)
domain_names = json.loads(output)['value']
print('VMS={}'.format(domain_names))
print('LEN VMS={}'.format(len(domain_names)))
cmd = "terraform output -json vm_ips"
[ret, output] = libutils.execute_bash_cmd(cmd)
domain_ips = json.loads(output)['value']
print('IPS={}'.format(domain_ips))
# decode json stuff here
i = 0
while i < len(domain_names):
domains.append(Domain(domain_names[i], domain_ips[i][0]))
print("NAME = {}".format(domain_names[i]))
print("IP = {}".format(domain_ips[i]))
i += 1
return domains
def snapshot(self, action):
"""
Create a snapshot of the domain
If desired, snapshots of a domain can be created when the environment
is freshly deployed. Thus, tests can reset the environment at any point
to revert the state of a domain as it was right after boot.
"""
if (action == 'create'):
cmd = ("virsh snapshot-create-as {} --name {}-snapshot".format(
self.name, self.name))
elif (action == 'delete'):
cmd = ("virsh snapshot-delete {} --current".format(
self.name, self.name))
elif (action == 'revert'):
cmd = ("virsh snapshot-revert {} --current".format(
self.name, self.name))
try:
libutils.execute_bash_cmd(cmd)
except libutils.TrfmCommandFailed as e:
self.logger.error("Failed to {} snapshot of domain {}.".format(
action, self.name))
raise libutils.TrfmSnapshotFailed(e)
def deploy(self, snapshots=False):
""" Deploy Environment
It creates the Terraform environment from the given .tf file
If snapshots == True, after the domains are up, it will create a
snapshot for each domain in case they are needed to be reverted
at a certain point of the test flow.
"""
self.logger.info("Deploying Terraform Environment")
[ret, output] = libutils.execute_bash_cmd('terraform init')
if ret != 0:
self.logger.error("There has been a problem"
" initializing terraform")
self.clean()
sys.exit(1)
[ret,
output] = (libutils.execute_bash_cmd('terraform apply -auto-approve'))
if ret != 0:
self.clean()
else:
self.deployed = True
self.domains = self.get_domains()
def get_network():
"""
Find a non-used network in the system
To allow multiple environments co-exist, network ranges can't be
hardcoded. Otherwise, new libvirt virtual networks can't be created.
This method offers a network range which is not currently used in the
range 10.0.0.0/24.
The second octet is a random number between 1 and 254.
The third octet is iterated from 1 to 254 until there is a non-used
range in the sytem.
"""
[ret, output] = libutils.execute_bash_cmd('ip route')
x = random.randint(1, 254)
y = 1
while y < 255:
if "10.{}.{}.0".format(x, y) in output:
y += 1
else:
break
if y == 255:
raise Exception("Cannot find available network range")
return "10.{}.{}.0/24".format(x, y)
def get_output(self, variable):
output = libutils.execute_bash_cmd("terraform output -json",
cwd=self.workdir)
return json.loads(output)[variable]['value'][0]
