def add_to_index(key, path): key = key or mod_name if key in test_index: display("COLLISION: Test '{}' Attempted '{}' Existing '{}'".format( key, test_index[key], path)) else: test_index[key] = path
def get_current_profile(args): import colorama colorama.init(autoreset=True) try: fore_red = colorama.Fore.RED if not IS_WINDOWS else '' fore_reset = colorama.Fore.RESET if not IS_WINDOWS else '' current_profile = check_output( shlex.split('az cloud show --query profile -otsv'), shell=IS_WINDOWS).decode('utf-8').strip() if not args.profile or current_profile == args.profile: args.profile = current_profile display( 'The tests are set to run against current profile {}.'.format( fore_red + current_profile + fore_reset)) elif current_profile != args.profile: display( 'The tests are set to run against profile {} but the current az cloud profile is {}.' .format(fore_red + args.profile + fore_reset, fore_red + current_profile + fore_reset)) display('SWITCHING TO PROFILE {}.'.format(fore_red + args.profile + fore_reset)) display('az cloud update --profile {}'.format(args.profile)) check_output(shlex.split('az cloud update --profile {}'.format( args.profile)), shell=IS_WINDOWS) return current_profile except CalledProcessError: display('Failed to retrieve current az profile') sys.exit(2)
def validate_usage(args): """ Ensure conflicting options aren't specified. """ test_usage = '[--test TESTS [TESTS ...]] [--src-file FILENAME]' ci_usage = '--ci' usages = [] if args.tests or args.src_file: usages.append(test_usage) if args.ci: usages.append(ci_usage) if len(usages) > 1: display('usage error: ' + ' | '.join(usages)) sys.exit(1)
def validate_usage(args): """ Ensure conflicting options aren't specified. """ test_usage = '[--test TESTS [TESTS ...]] [--src-file FILENAME]' ci_usage = '--ci' usages = [] if args.tests or args.src_file: usages.append(test_usage) if args.ci: usages.append(ci_usage) if len(usages) > 1: display('usage error: ' + ' | '.join(usages)) sys.exit(1)
def get_current_profile(args): try: fore_red = Fore.RED if not IS_WINDOWS else '' fore_reset = Fore.RESET if not IS_WINDOWS else '' current_profile = check_output( shlex.split('az cloud show --query profile -otsv'), shell=IS_WINDOWS).decode('utf-8').strip() if not args.profile: args.profile = current_profile display( 'The tests are set to run against current profile {}.'.format( fore_red + current_profile + fore_reset)) elif current_profile != args.profile: display( 'The tests are set to run against profile {} but the current az cloud profile is {}.' .format(fore_red + args.profile + fore_reset, fore_red + current_profile + fore_reset)) display( 'Please use "az cloud set" command to change the current profile.' ) sys.exit(1) return current_profile except CalledProcessError: display('Failed to retrieve current az profile') sys.exit(2)
def run_tests(modules, parallel, run_live, tests): if not modules and not tests: display('No tests set to run.') sys.exit(1) display(""" ============= Run Tests ============= """) if modules: display('Modules: {}'.format(', '.join(name for name, _, _ in modules))) # set environment variable if run_live: os.environ['AZURE_TEST_RUN_LIVE'] = 'True' test_paths = tests or [p for _, _, p in modules] display('Drive test by nosetests') from six import StringIO old_stderr = sys.stderr test_stderr = StringIO() sys.stderr = test_stderr runner = get_nose_runner(parallel=parallel, process_timeout=3600 if run_live else 600) results = runner([path for path in test_paths]) stderr_val = test_stderr.getvalue() sys.stderr = old_stderr test_stderr.close() failed_tests = summarize_tests(stderr_val) return results, failed_tests
def run_tests(modules, parallel, run_live, tests): if not modules and not tests: display('No tests set to run.') sys.exit(1) display(""" ============= Run Tests ============= """) if modules: display('Modules: {}'.format(', '.join(name for name, _, _ in modules))) # set environment variable if run_live: os.environ['AZURE_TEST_RUN_LIVE'] = 'True' test_paths = tests or [p for _, _, p in modules] display('Drive test by nosetests') from six import StringIO old_stderr = sys.stderr test_stderr = StringIO() sys.stderr = test_stderr runner = get_nose_runner(parallel=parallel, process_timeout=3600 if run_live else 600) results = runner([path for path in test_paths]) stderr_val = test_stderr.getvalue() sys.stderr = old_stderr test_stderr.close() failed_tests = summarize_tests(stderr_val) return results, failed_tests
def run_tests(modules, parallel, run_live, tests): if not modules and not tests: display('No tests set to run.') sys.exit(1) display(""" ============= Run Tests ============= """) if modules: display('Modules: {}'.format(', '.join(name for name, _, _ in modules))) # set environment variable if run_live: os.environ['AZURE_TEST_RUN_LIVE'] = 'True' test_paths = tests or [p for _, _, p in modules] display('Drive test by nosetests') runner = get_nose_runner(parallel=parallel, process_timeout=3600 if run_live else 600) results = runner([path for path in test_paths]) return results, []
def get_test_index(args): test_index_path = os.path.join(get_config_dir(), TEST_INDEX_FORMAT.format(args.profile)) test_index = {} if args.discover: test_index = discover_tests(args) with open(test_index_path, 'w') as f: f.write(json.dumps(test_index)) elif os.path.isfile(test_index_path): with open(test_index_path, 'r') as f: test_index = json.loads(''.join(f.readlines())) else: display('No test index found. Building test') test_index = discover_tests(args) with open(test_index_path, 'w') as f: f.write(json.dumps(test_index)) return test_index
def get_test_index(args): test_index_path = os.path.join(get_config_dir(), TEST_INDEX_FILE) test_index = {} if args.discover: test_index = discover_tests(args) with open(test_index_path, 'w') as f: f.write(json.dumps(test_index)) elif os.path.isfile(test_index_path): with open(test_index_path, 'r') as f: test_index = json.loads(''.join(f.readlines())) else: display('No test index found. Building test') test_index = discover_tests(args) with open(test_index_path, 'w') as f: f.write(json.dumps(test_index)) return test_index
def add_to_index(key, path): key = key or mod_name if key in test_index: if key not in conflicted_keys: conflicted_keys.append(key) mod1 = extract_module_name(path) mod2 = extract_module_name(test_index[key]) if mod1 != mod2: # resolve conflicted keys by prefixing with the module name and a dot (.) display("\nCOLLISION: Test '{}' exists in both '{}' and '{}'. Resolve using <MOD_NAME>.<NAME>".format(key, mod1, mod2)) test_index['{}.{}'.format(mod1, key)] = path test_index['{}.{}'.format(mod2, key)] = test_index[key] else: display("\nERROR: Test '{}' exists twice in the '{}' module. Please rename one or both and re-run --discover.".format(key, mod1)) else: test_index[key] = path
def add_to_index(key, path): key = key or mod_name if key in test_index: if key not in conflicted_keys: conflicted_keys.append(key) mod1 = extract_module_name(path) mod2 = extract_module_name(test_index[key]) if mod1 != mod2: # resolve conflicted keys by prefixing with the module name and a dot (.) display("\nCOLLISION: Test '{}' exists in both '{}' and '{}'. Resolve using <MOD_NAME>.<NAME>".format(key, mod1, mod2)) test_index['{}.{}'.format(mod1, key)] = path test_index['{}.{}'.format(mod2, key)] = test_index[key] else: display("\nERROR: Test '{}' exists twice in the '{}' module. Please rename one or both and re-run --discover.".format(key, mod1)) else: test_index[key] = path
def get_extension_modules(): from importlib import import_module import pkgutil from azure.cli.core.extension import get_extensions, get_extension_path, get_extension_modname extension_whls = get_extensions() ext_modules = [] if extension_whls: for ext_name in [e.name for e in extension_whls]: ext_dir = get_extension_path(ext_name) sys.path.append(ext_dir) try: ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) module = import_module(ext_mod) setattr(module, 'path', module.__path__[0]) ext_modules.append((module, ext_mod)) except Exception as ex: display("Error importing '{}' extension: {}".format(ext_mod, ex)) return ext_modules
def get_extension_modules(): from importlib import import_module import pkgutil from azure.cli.core.extension import get_extensions, get_extension_path, get_extension_modname extension_whls = get_extensions() ext_modules = [] if extension_whls: for ext_name in [e.name for e in extension_whls]: ext_dir = get_extension_path(ext_name) sys.path.append(ext_dir) try: ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) module = import_module(ext_mod) setattr(module, 'path', module.__path__[0]) ext_modules.append((module, ext_mod)) except Exception as ex: display("Error importing '{}' extension: {}".format( ext_mod, ex)) return ext_modules
def get_current_profile(args): import colorama colorama.init(autoreset=True) try: fore_red = colorama.Fore.RED if not IS_WINDOWS else '' fore_reset = colorama.Fore.RESET if not IS_WINDOWS else '' current_profile = check_output(shlex.split('az cloud show --query profile -otsv'), shell=IS_WINDOWS).decode('utf-8').strip() if not args.profile or current_profile == args.profile: args.profile = current_profile display('The tests are set to run against current profile {}.' .format(fore_red + current_profile + fore_reset)) elif current_profile != args.profile: display('The tests are set to run against profile {} but the current az cloud profile is {}.' .format(fore_red + args.profile + fore_reset, fore_red + current_profile + fore_reset)) display('SWITCHING TO PROFILE {}.'.format(fore_red + args.profile + fore_reset)) display('az cloud update --profile {}'.format(args.profile)) check_output(shlex.split('az cloud update --profile {}'.format(args.profile)), shell=IS_WINDOWS) return current_profile except CalledProcessError: display('Failed to retrieve current az profile') sys.exit(2)
def execute(args): from .main import run_tests validate_usage(args) current_profile = get_current_profile(args) test_index = get_test_index(args) modules = [] if args.ci: # CI Mode runs specific modules output('Running in CI Mode') selected_modules = [('All modules', 'azure.cli', 'azure.cli'), ('CLI Linter', 'automation.cli_linter', 'automation.cli_linter')] elif not (args.tests or args.src_file): # Default is to run with modules (possibly via environment variable) if os.environ.get('AZURE_CLI_TEST_MODULES', None): display('Test modules list is parsed from environment variable AZURE_CLI_TEST_MODULES.') modules = [m.strip() for m in os.environ.get('AZURE_CLI_TEST_MODULES').split(',')] selected_modules = filter_user_selected_modules_with_tests(modules, args.profile) if not selected_modules: display('\nNo tests selected.') sys.exit(1) else: # Otherwise run specific tests args.tests = args.tests or [] # Add any tests from file if args.src_file: with open(args.src_file, 'r') as f: for line in f.readlines(): line = line.strip('\r\n') line = line.strip('\n') if line not in args.tests: args.tests.append(line) test_paths = [] selected_modules = [] for t in args.tests: try: test_path = os.path.normpath(test_index[t]) mod_name = extract_module_name(test_path) test_paths.append(test_path) if mod_name not in selected_modules: selected_modules.append(mod_name) except KeyError: display("Test '{}' not found.".format(t)) continue selected_modules = filter_user_selected_modules_with_tests(selected_modules, args.profile) args.tests = test_paths success, failed_tests = run_tests(selected_modules, parallel=args.parallel, run_live=args.live, tests=args.tests) # if args.dest_file: # with open(args.dest_file, 'w') as f: # for failed_test in failed_tests: # f.write(failed_test + '\n') sys.exit(0 if success else 1)
def execute(args): from .main import run_tests, collect_test validate_usage(args) current_profile = get_current_profile(args) test_index = get_test_index(args) modules = [] if args.ci: # CI Mode runs specific modules selected_modules = [('CI mode', 'azure.cli', 'azure.cli')] elif not (args.tests or args.src_file): # Default is to run with modules (possibly via environment variable) if os.environ.get('AZURE_CLI_TEST_MODULES', None): display('Test modules list is parsed from environment variable AZURE_CLI_TEST_MODULES.') modules = [m.strip() for m in os.environ.get('AZURE_CLI_TEST_MODULES').split(',')] selected_modules = filter_user_selected_modules_with_tests(modules, args.profile) if not selected_modules: display('\nNo tests selected.') sys.exit(1) else: # Otherwise run specific tests args.tests = args.tests or [] # Add any tests from file if args.src_file: with open(args.src_file, 'r') as f: for line in f.readlines(): line = line.strip('\r\n') line = line.strip('\n') if line not in args.tests: args.tests.append(line) test_paths = [] selected_modules = [] for t in args.tests: try: test_path = os.path.normpath(test_index[t]) mod_name = extract_module_name(test_path) test_paths.append(test_path) if mod_name not in selected_modules: selected_modules.append(mod_name) except KeyError: display("Test '{}' not found.".format(t)) continue selected_modules = filter_user_selected_modules_with_tests(selected_modules, args.profile) args.tests = test_paths success, failed_tests = run_tests(selected_modules, parallel=args.parallel, run_live=args.live, tests=args.tests) if args.dest_file: with open(args.dest_file, 'w') as f: for failed_test in failed_tests: f.write(failed_test + '\n') sys.exit(0 if success else 1)
def run_tests(modules, parallel, run_live, tests): if not modules and not tests: display('No tests set to run.') sys.exit(1) display(""" ============= Run Tests ============= """) if modules: display('Modules: {}'.format(', '.join(name for name, _, _ in modules))) # set environment variable if run_live: os.environ['AZURE_TEST_RUN_LIVE'] = 'True' test_paths = tests or [p for _, _, p in modules] display('Drive test by nosetests') runner = get_nose_runner(parallel=parallel, process_timeout=3600 if run_live else 600) results = runner([path for path in test_paths]) return results, []
def summarize_tests(test_output): display(test_output) failed_tests = [] for line in test_output.splitlines(): if '... ERROR' in line or '... FAIL' in line: line = line.replace('(', '') line = line.replace(')', '') try: test_name, _, _, _ = line.split(' ') line = test_name except: pass failed_tests.append(line) if failed_tests: display(""" ========== FAILED ========== """) for failed_test in failed_tests: display(failed_test) return failed_tests
def summarize_tests(test_output): display(test_output) failed_tests = [] for line in test_output.splitlines(): if '... ERROR' in line or '... FAIL' in line: line = line.replace('(', '') line = line.replace(')', '') try: test_name, _, _, _ = line.split(' ') line = test_name except: pass failed_tests.append(line) if failed_tests: display(""" ========== FAILED ========== """) for failed_test in failed_tests: display(failed_test) return failed_tests
def discover_tests(args): """ Builds an index of tests so that the user can simply supply the name they wish to test instead of the full path. """ from importlib import import_module import os import pkgutil CORE_EXCLUSIONS = ['command_modules', '__main__', 'testsdk'] profile = args.profile mods_ns_pkg = import_module('azure.cli.command_modules') core_ns_pkg = import_module('azure.cli') command_modules = list(pkgutil.iter_modules(mods_ns_pkg.__path__)) core_modules = list(pkgutil.iter_modules(core_ns_pkg.__path__)) all_modules = command_modules + [ x for x in core_modules if x[1] not in CORE_EXCLUSIONS ] display(""" ================== Discover Tests ================== """) module_data = {} for mod in all_modules: mod_name = mod[1] if mod_name == 'core': mod_data = { 'filepath': os.path.join(mod[0].path, mod_name, 'tests'), 'base_path': 'azure.cli.{}.tests'.format(mod_name), 'files': {} } else: mod_data = { 'filepath': os.path.join(mod[0].path, mod_name, 'tests', profile), 'base_path': 'azure.cli.command_modules.{}.tests.{}'.format( mod_name, profile), 'files': {} } # get the list of test files in each module try: contents = os.listdir(mod_data['filepath']) test_files = { x[:-len('.py')]: {} for x in contents if x.startswith('test_') and x.endswith('.py') } except Exception: # skip modules that don't have tests display("Module '{}' has no tests.".format(mod_name)) continue for file_name in test_files: mod_data['files'][file_name] = {} test_file_path = mod_data['base_path'] + '.' + file_name try: module = import_module(test_file_path) except ImportError: display('Unable to import {}'.format(test_file_path)) continue module_dict = module.__dict__ classes = {} possible_test_classes = { x: y for x, y in module_dict.items() if not x.startswith('_') } for class_name, class_def in possible_test_classes.items(): try: class_dict = class_def.__dict__ except AttributeError: # skip non-class symbols in files like constants, imported methods, etc. continue if class_dict.get('__module__') == test_file_path: tests = [ x for x in class_def.__dict__ if x.startswith('test_') ] if tests: mod_data['files'][file_name][class_name] = tests module_data[mod_name] = mod_data test_index = {} def add_to_index(key, path): key = key or mod_name if key in test_index: display("COLLISION: Test '{}' Attempted '{}' Existing '{}'".format( key, test_index[key], path)) else: test_index[key] = path # build the index for mod_name, mod_data in module_data.items(): mod_path = mod_data['filepath'] for file_name, file_data in mod_data['files'].items(): file_path = os.path.join(mod_path, file_name) + '.py' for class_name, test_list in file_data.items(): for test_name in test_list: test_path = '{}:{}.{}'.format(file_path, class_name, test_name) add_to_index(test_name, test_path) class_path = '{}:{}'.format(file_path, class_name) add_to_index(class_name, class_path) add_to_index(file_name, file_path) add_to_index(mod_name, mod_path) return test_index
def execute(args): from .main import run_tests validate_usage(args) current_profile = get_current_profile(args) test_index = get_test_index(args) modules = [] if args.ci: # CI Mode runs specific modules output('Running in CI Mode') selected_modules = [('All modules', 'azure.cli', 'azure.cli'), ('CLI Linter', 'automation.cli_linter', 'automation.cli_linter')] modified_modules = _extract_top_level_modified_modules() if any(base_mod in modified_modules for base_mod in ['core', 'testsdk', 'telemetry']): pass # if modified modules contains those 3 modules, run all tests else: test_paths = set() for mod in modified_modules: try: test_paths.add(os.path.normpath(test_index[mod])) except KeyError: display("no tests found in module: {}".format(mod)) args.tests = test_paths selected_modules = [] elif not (args.tests or args.src_file): # Default is to run with modules (possibly via environment variable) if os.environ.get('AZURE_CLI_TEST_MODULES', None): display( 'Test modules list is parsed from environment variable AZURE_CLI_TEST_MODULES.' ) modules = [ m.strip() for m in os.environ.get('AZURE_CLI_TEST_MODULES').split(',') ] selected_modules = filter_user_selected_modules_with_tests( modules, args.profile) if not selected_modules: display('\nNo tests selected.') sys.exit(1) else: # Otherwise run specific tests args.tests = args.tests or [] # Add any tests from file if args.src_file: with open(args.src_file, 'r') as f: for line in f.readlines(): line = line.strip('\r\n') line = line.strip('\n') if line not in args.tests: args.tests.append(line) test_paths = [] selected_modules = [] for t in args.tests: try: test_path = os.path.normpath(test_index[t]) mod_name = extract_module_name(test_path) test_paths.append(test_path) if mod_name not in selected_modules: selected_modules.append(mod_name) except KeyError: display("Test '{}' not found.".format(t)) continue selected_modules = filter_user_selected_modules_with_tests( selected_modules, args.profile) args.tests = test_paths success, failed_tests = run_tests(selected_modules, parallel=args.parallel, run_live=args.live, tests=args.tests) # if args.dest_file: # with open(args.dest_file, 'w') as f: # for failed_test in failed_tests: # f.write(failed_test + '\n') sys.exit(0 if success else 1)
def discover_tests(args): """ Builds an index of tests so that the user can simply supply the name they wish to test instead of the full path. """ from importlib import import_module import pkgutil CORE_EXCLUSIONS = ['command_modules', '__main__', 'testsdk'] profile_split = args.profile.split('-') profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) mods_ns_pkg = import_module('azure.cli.command_modules') core_ns_pkg = import_module('azure.cli') command_modules = list(pkgutil.iter_modules(mods_ns_pkg.__path__)) core_modules = list(pkgutil.iter_modules(core_ns_pkg.__path__)) extensions = get_extension_modules() all_modules = command_modules + [ x for x in core_modules if x[1] not in CORE_EXCLUSIONS ] + extensions display(""" ================== Discover Tests ================== """) module_data = {} for mod in all_modules: mod_name = mod[1] if mod_name == 'core' or mod_name == 'telemetry': mod_data = { 'filepath': os.path.join(mod[0].path, mod_name, 'tests'), 'base_path': 'azure.cli.{}.tests'.format(mod_name), 'files': {} } elif mod_name.startswith('azext_'): mod_data = { 'filepath': os.path.join(mod[0].path, 'tests', profile_namespace), 'base_path': '{}.tests.{}'.format(mod_name, profile_namespace), 'files': {} } else: mod_data = { 'filepath': os.path.join(mod[0].path, mod_name, 'tests', profile_namespace), 'base_path': 'azure.cli.command_modules.{}.tests.{}'.format( mod_name, profile_namespace), 'files': {} } # get the list of test files in each module try: contents = os.listdir(mod_data['filepath']) test_files = { x[:-len('.py')]: {} for x in contents if x.startswith('test_') and x.endswith('.py') } except Exception: # skip modules that don't have tests display("Module '{}' has no tests.".format(mod_name)) continue for file_name in test_files: mod_data['files'][file_name] = {} test_file_path = mod_data['base_path'] + '.' + file_name try: module = import_module(test_file_path) except ImportError as ex: display('Unable to import {}. Reason: {}'.format( test_file_path, ex)) continue module_dict = module.__dict__ possible_test_classes = { x: y for x, y in module_dict.items() if not x.startswith('_') } for class_name, class_def in possible_test_classes.items(): try: class_dict = class_def.__dict__ except AttributeError: # skip non-class symbols in files like constants, imported methods, etc. continue if class_dict.get('__module__') == test_file_path: tests = [ x for x in class_def.__dict__ if x.startswith('test_') ] if tests: mod_data['files'][file_name][class_name] = tests module_data[mod_name] = mod_data test_index = {} conflicted_keys = [] def add_to_index(key, path): key = key or mod_name if key in test_index: if key not in conflicted_keys: conflicted_keys.append(key) mod1 = extract_module_name(path) mod2 = extract_module_name(test_index[key]) if mod1 != mod2: # resolve conflicted keys by prefixing with the module name and a dot (.) display( "\nCOLLISION: Test '{}' exists in both '{}' and '{}'. Resolve using <MOD_NAME>.<NAME>" .format(key, mod1, mod2)) test_index['{}.{}'.format(mod1, key)] = path test_index['{}.{}'.format(mod2, key)] = test_index[key] else: display( "\nERROR: Test '{}' exists twice in the '{}' module. Please rename one or both and re-run --discover." .format(key, mod1)) else: test_index[key] = path # build the index for mod_name, mod_data in module_data.items(): mod_path = mod_data['filepath'] for file_name, file_data in mod_data['files'].items(): file_path = os.path.join(mod_path, file_name) + '.py' for class_name, test_list in file_data.items(): for test_name in test_list: test_path = '{}:{}.{}'.format(file_path, class_name, test_name) add_to_index(test_name, test_path) class_path = '{}:{}'.format(file_path, class_name) add_to_index(class_name, class_path) add_to_index(file_name, file_path) add_to_index(mod_name, mod_path) # remove the conflicted keys since they would arbitrarily point to a random implementation for key in conflicted_keys: del test_index[key] return test_index
def discover_tests(args): """ Builds an index of tests so that the user can simply supply the name they wish to test instead of the full path. """ from importlib import import_module import pkgutil CORE_EXCLUSIONS = ['command_modules', '__main__', 'testsdk'] profile_split = args.profile.split('-') profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) mods_ns_pkg = import_module('azure.cli.command_modules') core_ns_pkg = import_module('azure.cli') command_modules = list(pkgutil.iter_modules(mods_ns_pkg.__path__)) core_modules = list(pkgutil.iter_modules(core_ns_pkg.__path__)) extensions = get_extension_modules() all_modules = command_modules + [x for x in core_modules if x[1] not in CORE_EXCLUSIONS] + extensions display(""" ================== Discover Tests ================== """) module_data = {} for mod in all_modules: mod_name = mod[1] if mod_name == 'core' or mod_name == 'telemetry': mod_data = { 'filepath': os.path.join(mod[0].path, mod_name, 'tests'), 'base_path': 'azure.cli.{}.tests'.format(mod_name), 'files': {} } elif mod_name.startswith('azext_'): mod_data = { 'filepath': os.path.join(mod[0].path, 'tests', profile_namespace), 'base_path': '{}.tests.{}'.format(mod_name, profile_namespace), 'files': {} } else: mod_data = { 'filepath': os.path.join(mod[0].path, mod_name, 'tests', profile_namespace), 'base_path': 'azure.cli.command_modules.{}.tests.{}'.format(mod_name, profile_namespace), 'files': {} } # get the list of test files in each module try: contents = os.listdir(mod_data['filepath']) test_files = {x[:-len('.py')]: {} for x in contents if x.startswith('test_') and x.endswith('.py')} except Exception: # skip modules that don't have tests display("Module '{}' has no tests.".format(mod_name)) continue for file_name in test_files: mod_data['files'][file_name] = {} test_file_path = mod_data['base_path'] + '.' + file_name try: module = import_module(test_file_path) except ImportError as ex: display('Unable to import {}. Reason: {}'.format(test_file_path, ex)) continue module_dict = module.__dict__ possible_test_classes = {x: y for x, y in module_dict.items() if not x.startswith('_')} for class_name, class_def in possible_test_classes.items(): try: class_dict = class_def.__dict__ except AttributeError: # skip non-class symbols in files like constants, imported methods, etc. continue if class_dict.get('__module__') == test_file_path: tests = [x for x in class_def.__dict__ if x.startswith('test_')] if tests: mod_data['files'][file_name][class_name] = tests module_data[mod_name] = mod_data test_index = {} conflicted_keys = [] def add_to_index(key, path): key = key or mod_name if key in test_index: if key not in conflicted_keys: conflicted_keys.append(key) mod1 = extract_module_name(path) mod2 = extract_module_name(test_index[key]) if mod1 != mod2: # resolve conflicted keys by prefixing with the module name and a dot (.) display("\nCOLLISION: Test '{}' exists in both '{}' and '{}'. Resolve using <MOD_NAME>.<NAME>".format(key, mod1, mod2)) test_index['{}.{}'.format(mod1, key)] = path test_index['{}.{}'.format(mod2, key)] = test_index[key] else: display("\nERROR: Test '{}' exists twice in the '{}' module. Please rename one or both and re-run --discover.".format(key, mod1)) else: test_index[key] = path # build the index for mod_name, mod_data in module_data.items(): mod_path = mod_data['filepath'] for file_name, file_data in mod_data['files'].items(): file_path = os.path.join(mod_path, file_name) + '.py' for class_name, test_list in file_data.items(): for test_name in test_list: test_path = '{}:{}.{}'.format(file_path, class_name, test_name) add_to_index(test_name, test_path) class_path = '{}:{}'.format(file_path, class_name) add_to_index(class_name, class_path) add_to_index(file_name, file_path) add_to_index(mod_name, mod_path) # remove the conflicted keys since they would arbitrarily point to a random implementation for key in conflicted_keys: del test_index[key] return test_index