def _ssh_close(self): if self._ssh_client: try: self._ssh_client.close() except SSHException: LOG.warning('Failed to close SSH connection.') self._ssh_client = None
def shutdown(self, wait=True): """Finds console log then stopping/deallocates VM""" LOG.debug('waiting on console log before stopping') attempts, exists = 5, False while not exists and attempts: try: attempts -= 1 exists = self.blob_client.get_blob_to_bytes( self.container, self.blob) LOG.debug('found console log') except Exception as e: if attempts: LOG.debug( 'Unable to find console log, ' '%s attempts remaining', attempts) sleep(15) else: LOG.warning('Could not find console log: %s', e) LOG.debug('stopping instance %s', self.image_id) vm_deallocate = \ self.platform.compute_client.virtual_machines.deallocate( self.platform.resource_group.name, self.image_id) if wait: vm_deallocate.wait()
def collect_test_data(args, snapshot, os_name, test_name): """Collect data for test case. @param args: cmdline arguments @param snapshot: instantiated snapshot @param test_name: name or path of test to run @return_value: tuple of results and fail count """ res = ({}, 1) # load test config test_name = config.path_to_name(test_name) test_config = config.load_test_config(test_name) user_data = test_config['cloud_config'] test_scripts = test_config['collect_scripts'] test_output_dir = os.sep.join( (args.data_dir, snapshot.platform_name, os_name, test_name)) # if test is not enabled, skip and return 0 failures if not test_config.get('enabled', False): LOG.warning('test config %s is not enabled, skipping', test_name) return ({}, 0) # if testcase requires a feature flag that the image does not support, # skip the testcase with a warning req_features = test_config.get('required_features', []) if any(feature not in snapshot.features for feature in req_features): LOG.warning('test config %s requires features not supported by image, ' 'skipping.\nrequired features: %s\nsupported features: %s', test_name, req_features, snapshot.features) return ({}, 0) # if there are user data overrides required for this test case, apply them overrides = snapshot.config.get('user_data_overrides', {}) if overrides: LOG.debug('updating user data for collect with: %s', overrides) user_data = util.update_user_data(user_data, overrides) # create test instance component = PlatformComponent( partial(platforms.get_instance, snapshot, user_data, block=True, start=False, use_desc=test_name)) LOG.info('collecting test data for test: %s', test_name) with component as instance: start_call = partial(run_single, 'boot instance', partial( instance.start, wait=True, wait_for_cloud_init=True)) collect_calls = [partial(run_single, 'script {}'.format(script_name), partial(collect_script, instance, test_output_dir, script, script_name)) for script_name, script in test_scripts.items()] console_log = partial( run_single, 'collect console', partial(collect_console, instance, test_output_dir)) res = run_stage('collect for test: {}'.format(test_name), [start_call] + collect_calls + [console_log]) return res
def normalize_create_args(args): """Normalize CREATE arguments. @param args: parsed args @return_value: updated args, or None if errors occurred """ # ensure valid name for new test if len(args.name.split('/')) != 2: LOG.error('invalid test name: %s', args.name) return None if os.path.exists(config.name_to_path(args.name)): msg = 'test: {} already exists'.format(args.name) if args.force: LOG.warning('%s but ignoring due to --force', msg) else: LOG.error(msg) return None # ensure test config valid if specified if isinstance(args.config, str) and len(args.config) == 0: LOG.error('test config cannot be empty if specified') return None # ensure description valid if specified if (isinstance(args.description, str) and (len(args.description) > 70 or len(args.description) == 0)): LOG.error('test description must be between 1 and 70 characters') return None return args
def collect_test_data(args, snapshot, os_name, test_name): """ collect data for test case args: cmdline arguments snapshot: instantiated snapshot test_name: name or path of test to run return_value: tuple of results and fail count """ res = ({}, 1) # load test config test_name = config.path_to_name(test_name) test_config = config.load_test_config(test_name) user_data = test_config['cloud_config'] test_scripts = test_config['collect_scripts'] test_output_dir = os.sep.join( (args.data_dir, snapshot.platform_name, os_name, test_name)) boot_timeout = (test_config.get('boot_timeout') if isinstance( test_config.get('boot_timeout'), int) else snapshot.config.get('timeout')) # if test is not enabled, skip and return 0 failures if not test_config.get('enabled', False): LOG.warning('test config %s is not enabled, skipping', test_name) return ({}, 0) # create test instance component = PlatformComponent( partial(instances.get_instance, snapshot, user_data, block=True, start=False, use_desc=test_name)) LOG.info('collecting test data for test: %s', test_name) with component as instance: start_call = partial( run_single, 'boot instance', partial(instance.start, wait=True, wait_time=boot_timeout)) collect_calls = [ partial( run_single, 'script {}'.format(script_name), partial(collect_script, instance, test_output_dir, script, script_name)) for script_name, script in test_scripts.items() ] res = run_stage('collect for test: {}'.format(test_name), [start_call] + collect_calls) return res
def verify_data(data_dir, platform, os_name, tests): """Verify test data is correct. @param data_dir: top level directory for all tests @param platform: The platform name we for this test data (e.g. lxd) @param os_name: The operating system under test (xenial, artful, etc.). @param tests: list of test names @return_value: {<test_name>: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) # get cloudconfig for test test_conf = config.load_test_config(test_name) test_module = config.name_to_module(test_name) cloud_conf = test_conf['cloud_config'] # load script outputs data = {'platform': platform, 'os_name': os_name} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): with open(os.path.join(test_dir, script_name), 'rb') as fp: data[script_name] = fp.read() # get test suite and launch tests suite = testcases.get_suite(test_module, data, cloud_conf) suite_results = runner.run(suite) res[test_name] = { 'passed': suite_results.wasSuccessful(), 'failures': [{ 'module': type(test_class).__base__.__module__, 'class': type(test_class).__base__.__name__, 'function': str(test_class).split()[0], 'error': trace.splitlines()[-1], 'traceback': trace, } for test_class, trace in suite_results.failures] } for failure in res[test_name]['failures']: LOG.warning('test case: %s failed %s.%s with: %s', test_name, failure['class'], failure['function'], failure['error']) return res
def verify_data(base_dir, tests): """ verify test data is correct, base_dir: base directory for data test_config: dict of all test config, from util.load_test_config() tests: list of test names return_value: {<test_name>: {passed: True/False, failures: []}} """ runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) # get cloudconfig for test test_conf = config.load_test_config(test_name) test_module = config.name_to_module(test_name) cloud_conf = test_conf['cloud_config'] # load script outputs data = {} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): with open(os.path.join(test_dir, script_name), 'r') as fp: data[script_name] = fp.read() # get test suite and launch tests suite = testcases.get_suite(test_module, data, cloud_conf) suite_results = runner.run(suite) res[test_name] = { 'passed': suite_results.wasSuccessful(), 'failures': [{ 'module': type(test_class).__base__.__module__, 'class': type(test_class).__base__.__name__, 'function': str(test_class).split()[0], 'error': trace.splitlines()[-1], 'traceback': trace, } for test_class, trace in suite_results.failures] } for failure in res[test_name]['failures']: LOG.warning('test case: %s failed %s.%s with: %s', test_name, failure['class'], failure['function'], failure['error']) return res
def verify_data(data_dir, platform, os_name, tests): """Verify test data is correct. @param data_dir: top level directory for all tests @param platform: The platform name we for this test data (e.g. lxd) @param os_name: The operating system under test (xenial, artful, etc.). @param tests: list of test names @return_value: {<test_name>: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) # get cloudconfig for test test_conf = config.load_test_config(test_name) test_module = config.name_to_module(test_name) cloud_conf = test_conf['cloud_config'] # load script outputs data = {'platform': platform, 'os_name': os_name} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): with open(os.path.join(test_dir, script_name), 'rb') as fp: data[script_name] = fp.read() # get test suite and launch tests suite = testcases.get_suite(test_module, data, cloud_conf) suite_results = runner.run(suite) res[test_name] = { 'passed': suite_results.wasSuccessful(), 'failures': [{'module': type(test_class).__base__.__module__, 'class': type(test_class).__base__.__name__, 'function': str(test_class).split()[0], 'error': trace.splitlines()[-1], 'traceback': trace, } for test_class, trace in suite_results.failures] } for failure in res[test_name]['failures']: LOG.warning('test case: %s failed %s.%s with: %s', test_name, failure['class'], failure['function'], failure['error']) return res
def run(args): """ run full test suite """ failed = 0 args.data_dir = tempfile.mkdtemp(prefix='cloud_test_data_') LOG.debug('using tmpdir %s', args.data_dir) try: failed += collect.collect(args) failed += verify.verify(args) except Exception: failed += 1 raise finally: # TODO: make this configurable via environ or cmdline if failed: LOG.warning('some tests failed, leaving data in %s', args.data_dir) else: shutil.rmtree(args.data_dir) return failed
def shutdown(self, wait=True, retry=1): """Shutdown instance.""" if self.pylxd_container.status == 'Stopped': return try: LOG.debug("%s: shutting down (wait=%s)", self, wait) self.pylxd_container.stop(wait=wait) except (pylxd_exc.LXDAPIException, pylxd_exc.NotFound) as e: # An exception happens here sometimes (LP: #1783198) # LOG it, and try again. LOG.warning(("%s: shutdown(retry=%d) caught %s in shutdown " "(response=%s): %s"), self, retry, e.__class__.__name__, e.response, e) if isinstance(e, pylxd_exc.NotFound): LOG.debug("container_exists(%s) == %s", self.name, self.platform.container_exists(self.name)) if retry == 0: raise e return self.shutdown(wait=wait, retry=retry - 1)
def verify(args): """Verify test data. @param args: directory of test data @return_value: 0 for success, or number of failed tests """ failed = 0 res = {} # find test data tests = util.list_test_data(args.data_dir) for platform in tests.keys(): res[platform] = {} for os_name in tests[platform].keys(): test_name = "platform='{}', os='{}'".format(platform, os_name) LOG.info('test: %s verifying test data', test_name) # run test res[platform][os_name] = verify_data( os.sep.join((args.data_dir, platform, os_name)), tests[platform][os_name]) # handle results fail_list = [ k for k, v in res[platform][os_name].items() if not v.get('passed') ] if len(fail_list) == 0: LOG.info('test: %s passed all tests', test_name) else: LOG.warning('test: %s failed %s tests', test_name, len(fail_list)) failed += len(fail_list) # dump results LOG.debug('verify results: %s', res) if args.result: util.merge_results({'verify': res}, args.result) return failed
def shutdown(self, wait=True, retry=1): """Shutdown instance.""" if self.pylxd_container.status == 'Stopped': return try: LOG.debug("%s: shutting down (wait=%s)", self, wait) self.pylxd_container.stop(wait=wait) except (pylxd_exc.LXDAPIException, pylxd_exc.NotFound) as e: # An exception happens here sometimes (LP: #1783198) # LOG it, and try again. LOG.warning( ("%s: shutdown(retry=%d) caught %s in shutdown " "(response=%s): %s"), self, retry, e.__class__.__name__, e.response, e) if isinstance(e, pylxd_exc.NotFound): LOG.debug("container_exists(%s) == %s", self.name, self.platform.container_exists(self.name)) if retry == 0: raise e return self.shutdown(wait=wait, retry=retry - 1)
def verify(args): """Verify test data. @param args: directory of test data @return_value: 0 for success, or number of failed tests """ failed = 0 res = {} # find test data tests = util.list_test_data(args.data_dir) for platform in tests.keys(): res[platform] = {} for os_name in tests[platform].keys(): test_name = "platform='{}', os='{}'".format(platform, os_name) LOG.info('test: %s verifying test data', test_name) # run test res[platform][os_name] = verify_data( args.data_dir, platform, os_name, tests[platform][os_name]) # handle results fail_list = [k for k, v in res[platform][os_name].items() if not v.get('passed')] if len(fail_list) == 0: LOG.info('test: %s passed all tests', test_name) else: LOG.warning('test: %s failed %s tests', test_name, len(fail_list)) failed += len(fail_list) # dump results LOG.debug('\n---- Verify summarized results:\n%s', format_results(res)) if args.result: util.merge_results({'verify': res}, args.result) return failed
def _wait_for_system(self, wait_for_cloud_init): """Wait until system has fully booted and cloud-init has finished. @param wait_time: maximum time to wait @return_value: None, may raise OSError if wait_time exceeded """ def clean_test(test): """Clean formatting for system ready test testcase.""" return ' '.join(l for l in test.strip().splitlines() if not l.lstrip().startswith('#')) boot_timeout = self.config['boot_timeout'] tests = [self.config['system_ready_script']] if wait_for_cloud_init: tests.append(self.config['cloud_init_ready_script']) formatted_tests = ' && '.join(clean_test(t) for t in tests) cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' 'exit 0; sleep 1; done; exit 1').format(time=boot_timeout, test=formatted_tests) end_time = time.time() + boot_timeout while True: try: return_code = self.execute( cmd, rcs=(0, 1), description='wait for instance start' )[-1] if return_code == 0: break except util.InTargetExecuteError: LOG.warning("failed to connect via SSH") if time.time() < end_time: time.sleep(3) else: raise util.PlatformError('ssh', 'after %ss instance is not ' 'reachable' % boot_timeout)
def _wait_for_system(self, wait_for_cloud_init): """Wait until system has fully booted and cloud-init has finished. @param wait_time: maximum time to wait @return_value: None, may raise OSError if wait_time exceeded """ def clean_test(test): """Clean formatting for system ready test testcase.""" return ' '.join(line for line in test.strip().splitlines() if not line.lstrip().startswith('#')) boot_timeout = self.config['boot_timeout'] tests = [self.config['system_ready_script']] if wait_for_cloud_init: tests.append(self.config['cloud_init_ready_script']) formatted_tests = ' && '.join(clean_test(t) for t in tests) cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' 'exit 0; sleep 1; done; exit 1').format(time=boot_timeout, test=formatted_tests) end_time = time.time() + boot_timeout while True: try: return_code = self.execute( cmd, rcs=(0, 1), description='wait for instance start')[-1] if return_code == 0: break except util.InTargetExecuteError: LOG.warning("failed to connect via SSH") if time.time() < end_time: time.sleep(3) else: raise util.PlatformError( 'ssh', 'after %ss instance is not ' 'reachable' % boot_timeout)
def collect_test_data(args, snapshot, os_name, test_name): """Collect data for test case. @param args: cmdline arguments @param snapshot: instantiated snapshot @param test_name: name or path of test to run @return_value: tuple of results and fail count """ res = ({}, 1) # load test config test_name_in = test_name test_name = config.path_to_name(test_name) test_config = config.load_test_config(test_name) user_data = test_config['cloud_config'] test_scripts = test_config['collect_scripts'] test_output_dir = os.sep.join( (args.data_dir, snapshot.platform_name, os_name, test_name)) # if test is not enabled, skip and return 0 failures if not test_config.get('enabled', False): LOG.warning('test config %s is not enabled, skipping', test_name) return ({}, 0) test_class = get_test_class( config.name_to_module(test_name_in), test_data={'platform': snapshot.platform_name, 'os_name': os_name}, test_conf=test_config['cloud_config']) try: test_class.maybeSkipTest() except base.SkipTest as s: LOG.warning('skipping test config %s: %s', test_name, s) return ({}, 0) # if testcase requires a feature flag that the image does not support, # skip the testcase with a warning req_features = test_config.get('required_features', []) if any(feature not in snapshot.features for feature in req_features): LOG.warning('test config %s requires features not supported by image, ' 'skipping.\nrequired features: %s\nsupported features: %s', test_name, req_features, snapshot.features) return ({}, 0) # if there are user data overrides required for this test case, apply them overrides = snapshot.config.get('user_data_overrides', {}) if overrides: LOG.debug('updating user data for collect with: %s', overrides) user_data = util.update_user_data(user_data, overrides) # create test instance component = PlatformComponent( partial(platforms.get_instance, snapshot, user_data, block=True, start=False, use_desc=test_name), preserve_instance=args.preserve_instance) LOG.info('collecting test data for test: %s', test_name) with component as instance: start_call = partial(run_single, 'boot instance', partial( instance.start, wait=True, wait_for_cloud_init=True)) collect_calls = [partial(run_single, 'script {}'.format(script_name), partial(collect_script, instance, test_output_dir, script, script_name)) for script_name, script in test_scripts.items()] res = run_stage('collect for test: {}'.format(test_name), [start_call] + collect_calls) instance.shutdown() collect_console(instance, test_output_dir) return res