def can_ssh_to_server(vm_name, username='******'):
host = get_vm_info(vm_name)['ip']
key_filename = cvl_config.VM_KEY_LOCATION + '/' + cvl_config.VM_KEY_NAME
try:
client = ssh.SSHClient()
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
client.connect(host, username=username, key_filename=key_filename, timeout=10)
client.close()
return True
except:
return False
def test_boot_with_volume(self):
logger.info("Starting boot instance with volume test")
# create volume
logger.info("Creating new volume")
volume = self.cinder.volumes.create(1, display_name=Nova_health_tests.VOLUME_NAME)
bdm = {"/dev/vdb": "{0}:::0".format(volume.id)}
# boot instance
logger.info("Booting new instance")
flavor = self.nova.flavors.find(name=self.flavor)
image = self.nova.images.find(name=self.cirros_image)
server_id = self.nova.servers.create(
Nova_health_tests.INSTANCE_NAME, image=image, block_device_mapping=bdm, flavor=flavor
).id
newserver = poll_until(
lambda: self.nova.servers.get(server_id), lambda inst: inst.status != "BUILD", sleep_time=2
)
self.assertEquals("ACTIVE", newserver.status)
logger.info("Booted new instance: " + server_id)
client = ssh.SSHClient()
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
network = newserver.networks["private"][-1]
logger.info("SSHing to %s", network)
poll_until(
lambda: check_for_exception(client.connect, network, username="******", password="******"),
lambda result: result,
sleep_time=2,
)
_, stdout, _ = client.exec_command("ls /dev")
result = stdout.readlines()
client.close()
logger.info("Files in /dev: %s", result)
self.assertTrue("vdb\n" in result)
def test_boot_with_volume(self):
logger.info("-----------------------------------------------")
logger.info("Starting boot instance with volume test")
logger.info("-----------------------------------------------")
# Add keypair
logger.info("Adding new pub key")
keypair = self.nova.keypairs.create(self.KEY_NAME)
f = open(self.KEY_FILE_NAME, 'w')
f.write(keypair.private_key)
f.close()
# create sec group + rule
logger.info("Creating new security group + rules")
secgroup = self.nova.security_groups.create(self.SECGROUP_NAME,
'test_boot_with_volume')
self.nova.security_group_rules.create(secgroup.id, 'tcp', 22, 22,
'0.0.0.0/0')
# create volume
logger.info("Creating new volume")
volume = self.cinder.volumes.create(1, display_name=self.VOLUME_NAME)
logger.info("Volume created: " + volume.id)
bdm = {'/dev/vdb': '{0}:::0'.format(volume.id)}
# boot instance
logger.info("Booting new instance: %s", self.INSTANCE_NAME)
flavor = self.nova.flavors.find(name=self.non_eph_flavor)
image = self.nova.images.find(name=self.image)
server_id = self.nova.servers.create(self.INSTANCE_NAME,
image=image,
block_device_mapping=bdm,
key_name=self.KEY_NAME,
security_groups=[secgroup.name],
flavor=flavor,
availability_zone=self.availability_zone).id
logger.info("Instance %s is building", server_id)
newserver = poll_until(lambda: self.nova.servers.get(server_id),
lambda inst: inst.status != 'BUILD',
sleep_time=5)
self.assertEquals('ACTIVE', newserver.status)
logger.info("Instance %s is active", server_id)
logger.info(vars(newserver))
time.sleep(10) # sleep 10 seconds while ssh server is starting
client = ssh.SSHClient()
client.load_host_keys('/dev/null')
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
logger.info("Network label for instance %s: %s", server_id, newserver.networks)
network = newserver.networks[self.network_label][-1]
logger.info('SSHing to %s', network)
poll_until(lambda: check_for_exception(client.connect,
network,
username='******',
key_filename=self
.KEY_FILE_NAME),
lambda result: result,
sleep_time=5)
result = execute_ssh_command(client, 'ls /dev')
client.close()
logger.info('Files in /dev: %s', result)
self.assertTrue('vdb\n' in result)
def test_boot_with_ephemeral(self):
logger.info("-----------------------------------------------")
logger.info("Starting boot instance with ephemeral test")
logger.info("-----------------------------------------------")
# # Add keypair
# logger.info("Adding new pub key")
# keypair = self.nova.keypairs.create(self.KEY_NAME)
# f = open(self.KEY_FILE_NAME, 'w')
# f.write(keypair.private_key)
# f.close()
# boot instance
logger.info("Booting new instance: %s", self.INSTANCE_NAME)
flavor = self.nova.flavors.find(name=self.flavor)
image = self.nova.images.find(name=self.image)
server_id = self.nova.servers.create(self.INSTANCE_NAME,
image=image,
key_name='paas-racks-jenkins',
flavor=flavor,
availability_zone=self.availability_zone).id
logger.info("Instance %s is building", server_id)
newserver = None
try:
newserver = poll_until(lambda: self.nova.servers.get(server_id),
lambda inst: inst.status != 'BUILD',
sleep_time=5)
except PollTimeout as e:
resized_server = e.previous_result
error_message = 'Instance is stuck in BUILD.\n' \
'Instance ID = ' + server_id + '\n'
if resized_server is not None:
error_message = error_message + 'Host ID = ' + resized_server.hostId + '\n' \
'Network = ' + str(resized_server.networks) + '\n'
self.fail(error_message)
finally:
if newserver is not None:
logger.info(vars(newserver))
self.assertEquals('ACTIVE', newserver.status, 'The instance %(id)s is in %(status)s' % {'id': server_id, 'status': newserver.status})
logger.info("Instance %s is active", server_id)
# sleep 10 seconds for instance to be ready (ssh server)
time.sleep(10)
secGroupExists = False
groups = self.nova.security_groups.list()
for group in groups:
if group.name == self.SECGROUP_NAME:
secGroupExists = True
break
if not secGroupExists:
logger.info("Creating new security group + rules")
secgroup = self.nova.security_groups.create(self.SECGROUP_NAME,
'test_boot_with_ephemeral')
self.nova.security_group_rules.create(secgroup.id, 'tcp', 22, 22, '0.0.0.0/0')
self.nova.security_group_rules.create(secgroup.id, 'icmp', -1, -1, '0.0.0.0/0')
self.nova.servers.add_security_group(server_id, self.SECGROUP_NAME)
# Ensure secgroup is not open
logger.info("Network label for instance %s: %s", server_id, newserver.networks)
network = newserver.networks[self.network_label][-1]
logger.info('SSHing to %s', network)
client = ssh.SSHClient()
client.load_host_keys('/dev/null')
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
try:
poll_until(lambda: check_for_exception(client.connect,
network,
username='******',
key_filename=self
.KEY_FILE_NAME),
lambda result: result,
sleep_time=5)
logger.info('Check for /dev/vdb...')
_, stdout, _ = client.exec_command('ls /dev')
result = stdout.readlines()
self.assertTrue('vdb\n' in result, '/dev/vdb not found')
# Test outbound connectivity
logger.info("Pinging google.com...")
exit_status = 0
for x in range(0, 5):
_, stdout, stderr = client.exec_command('ping google.com -c 1')
while not stdout.channel.exit_status_ready():
if stdout.channel.recv_ready():
rl, wl, xl = select.select([stdout.channel], [], [], 0.0)
if len(rl) > 0:
# Print data from stdout
logger.info(stdout.channel.recv(1024))
if stderr.channel.recv_stderr_ready():
rl, wl, xl = select.select([stderr.channel], [], [], 0.0)
if len(rl) > 0:
# Print data from stderr
logger.error(stderr.channel.recv_stderr(1024))
exit_status = stdout.channel.recv_exit_status()
if exit_status == 0:
break
else:
# sleep for 5 secs to allow network to be ready
time.sleep(5)
self.assertEquals(0, exit_status, 'Ping to google.com failed!')
except PollTimeout:
error_message = 'Unable to SSH to ' + network + '\n' \
'Instance ID = ' + newserver.id + '\n' \
'Host ID = ' + newserver.hostId + '\n' \
'Status = ' + newserver.status + '\n' \
'Network = ' + str(newserver.networks) + '\n'
self.fail(error_message)
finally:
client.close()
def test_resize_flavor(self):
logger.info("-----------------------------------------------")
logger.info("Starting resize flavor test")
logger.info("-----------------------------------------------")
# Add keypair
logger.info("Adding new pub key")
keypair = self.nova.keypairs.create(self.KEY_NAME)
f = open(self.KEY_FILE_NAME, 'w')
f.write(keypair.private_key)
f.close()
# create sec group + rule
logger.info("Creating new security group + rules")
secgroup = self.nova.security_groups.create(self.SECGROUP_NAME,
'test_boot_with_ephemeral')
self.nova.security_group_rules.create(secgroup.id, 'tcp', 22, 22,
'0.0.0.0/0')
# boot instance
logger.info("Booting new instance: %s", self.INSTANCE_NAME)
small_flavor = self.nova.flavors.find(name=self.flavor)
medium_flavor = self.nova.flavors.find(name=self.flavor_2)
image = self.nova.images.find(name=self.image)
server_id = self.nova.servers.create(self.INSTANCE_NAME,
image=image,
flavor=small_flavor,
key_name=self.KEY_NAME,
security_groups=[secgroup.name],
availability_zone=self.availability_zone).id
logger.info("Instance %s is building", server_id)
newserver = None
try:
newserver = poll_until(lambda: self.nova.servers.get(server_id),
lambda inst: inst.status != 'BUILD',
sleep_time=5)
except PollTimeout as e:
newserver = e.previous_result
error_message = 'Instance is stuck in BUILD.\n' \
'Instance ID = ' + server_id + '\n'
if newserver is not None:
error_message = error_message + 'Host ID = ' + newserver.hostId + '\n'
self.fail(error_message)
finally:
if newserver is not None:
logger.info(vars(newserver))
self.assertEquals('ACTIVE', newserver.status, 'The instance %(id)s is in %(status)s' % {'id': server_id, 'status': newserver.status})
logger.info("Instance %s is active", server_id)
# sleep 10 seconds for instance to be ready (ssh server)
time.sleep(10)
# SSH into the instance
logger.info("Network label for instance %s: %s", server_id, newserver.networks)
network = newserver.networks[self.network_label][-1]
logger.info('SSHing to %s', network)
client = ssh.SSHClient()
client.load_host_keys('/dev/null')
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
try:
poll_until(lambda: check_for_exception(client.connect,
network,
username='******',
key_filename=self
.KEY_FILE_NAME),
lambda result: result,
sleep_time=5,
time_out=10 * MINUTE)
# Create new file in the instance
logger.info("Creating new file, test_file, in root partition of the instance...")
execute_ssh_command(client, "python -c \"import os; import datetime; f = open('/home/ubuntu/test_file','w'); f.write(str(datetime.datetime.now()) + '\\n') ; f.flush(); os.fsync(f.fileno()); f.close()\"")
result = execute_ssh_command(client, 'ls')
logger.info(result)
file_content_before_resize_root = execute_ssh_command(client, "cat test_file")
logger.info("Content of test_file: " + str(file_content_before_resize_root))
self.assertTrue('test_file\n' in result, 'Newly create file in root partition, test_file, not found')
logger.info("Creating new file, test_file_eph, in the ephemeral partition of the instance...")
execute_ssh_command(client, "sudo python -c \"import os; import datetime; f = open('/mnt/test_file_eph','w'); f.write(str(datetime.datetime.now()) + '\\n') ; f.flush(); os.fsync(f.fileno()); f.close()\"")
result = execute_ssh_command(client, 'ls /mnt')
logger.info(result)
file_content_before_resize_eph = execute_ssh_command(client, "cat /mnt/test_file_eph")
logger.info("Content of test_file_eph: " + str(file_content_before_resize_eph))
self.assertTrue('test_file_eph\n' in result, 'Newly create file in ephemeral partition, test_file_eph, not found')
except PollTimeout:
error_message = 'Unable to SSH to ' + network + '\n' \
'Instance ID = ' + newserver.id + '\n' \
'Host ID = ' + newserver.hostId + '\n' \
'Status = ' + newserver.status + '\n' \
'Network = ' + str(newserver.networks) + '\n'
self.fail(error_message)
finally:
client.close()
self.nova.servers.resize(newserver.id, medium_flavor)
logger.info("Resizing instance %s", server_id)
resized_server = None
try:
resized_server = poll_until(lambda: self.nova.servers.get(server_id),
lambda inst: inst.status != 'BUILD' and
inst.status != 'RESIZE',
sleep_time=10)
self.assertEquals('VERIFY_RESIZE', resized_server.status, 'The instance %(id)s is in %(status)s' % {'id': resized_server.id, 'status': resized_server.status})
except PollTimeout as e:
resized_server = e.previous_result
error_message = 'Instance is stuck in RESIZE.\n' \
'Instance ID = ' + server_id + '\n'
if resized_server is not None:
error_message = error_message + 'Host ID = ' + resized_server.hostId + '\n' \
'Status = ' + resized_server.status + '\n' \
'Network = ' + str(resized_server.networks) + '\n'
self.fail(error_message)
finally:
if resized_server is not None:
logger.info(vars(resized_server))
self.nova.servers.confirm_resize(newserver.id)
try:
resized_server = poll_until(lambda: self.nova.servers.get(server_id),
lambda inst: inst.status != 'BUILD' and
inst.status != 'VERIFY_RESIZE', sleep_time=5)
self.assertEquals('ACTIVE', resized_server.status, 'The instance %(id)s is in %(status)s' % {'id': resized_server.id, 'status': resized_server.status})
except PollTimeout as e:
resized_server = e.previous_result
error_message = 'Instance is never became ACTIVE after issuing a confirm_resize.\n' \
'Instance ID = ' + server_id + '\n'
if resized_server is not None:
error_message = error_message + 'Host ID = ' + resized_server.hostId + '\n' \
'Status = ' + resized_server.status + '\n' \
'Network = ' + str(resized_server.networks) + '\n'
self.fail(error_message)
self.assertEquals(medium_flavor.id, resized_server.flavor['id'], 'Flavor on resized instance do not match specified flavor')
logger.info("Network label for instance %s: %s", server_id, newserver.networks)
network = newserver.networks[self.network_label][-1]
# sleep 10 seconds while ssh server is starting
time.sleep(10)
logger.info('SSHing to %s', network)
client = ssh.SSHClient()
client.load_host_keys('/dev/null')
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
try:
poll_until(lambda: check_for_exception(client.connect,
network,
username='******',
key_filename=self
.KEY_FILE_NAME),
lambda result: result,
sleep_time=5,
time_out=10 * MINUTE)
# Testing if created file exists
logger.info("Verifying test_file exist in root partition of the resized instance...")
result = execute_ssh_command(client, 'ls')
logger.info(result)
file_content_after_resize_root = execute_ssh_command(client, "cat test_file")
logger.info("Content of test_file: " + str(file_content_after_resize_root))
self.assertTrue('test_file\n' in result, 'Test file that was created in the root partition before resize is not found after resize')
self.assertEquals(file_content_before_resize_root[0], file_content_after_resize_root[0], 'Content in files for data stored in ephemeral partition do not match')
logger.info("Verifying test_file exist in ephemeral partition of the resized instance...")
result = execute_ssh_command(client, 'sudo ls /mnt')
logger.info(result)
file_content_after_resize_eph = execute_ssh_command(client, "cat /mnt/test_file_eph")
logger.info("Content of test_file_eph: " + str(file_content_after_resize_eph))
self.assertTrue('test_file_eph\n' in result, 'Test file that was created in the ephemeral partition before resize is not found after resize')
self.assertEquals(file_content_before_resize_eph[0], file_content_after_resize_eph[0], 'Content in files for data stored in ephemeral partition do not match')
# Test outbound connectivity
logger.info("Pinging google.com...")
exit_status = 0
for x in range(0, 5):
_, stdout, stderr = client.exec_command('ping google.com -c 1')
while not stdout.channel.exit_status_ready():
if stdout.channel.recv_ready():
rl, wl, xl = select.select([stdout.channel], [], [], 0.0)
if len(rl) > 0:
# Print data from stdout
logger.info(stdout.channel.recv(1024))
if stderr.channel.recv_stderr_ready():
rl, wl, xl = select.select([stderr.channel], [], [], 0.0)
if len(rl) > 0:
# Print data from stderr
logger.error(stderr.channel.recv_stderr(1024))
exit_status = stdout.channel.recv_exit_status()
if exit_status == 0:
break
else:
# sleep for 5 secs to allow network to be ready
time.sleep(5)
self.assertEquals(0, exit_status, 'Ping to google.com failed!')
except PollTimeout:
error_message = 'Unable to SSH to ' + network + '\n' \
'Instance ID = ' + resized_server.id + '\n' \
'Host ID = ' + resized_server.hostId + '\n' \
'Status = ' + resized_server.status + '\n' \
'Network = ' + str(resized_server.networks) + '\n'
self.fail(error_message)
finally:
client.close()
