def detect_changes(args): """ :type args: TestConfig :rtype: list[str] | None """ if is_shippable(): display.info( 'Shippable detected, collecting parameters from environment.') paths = detect_changes_shippable(args) elif args.changed_from or args.changed_path: paths = args.changed_path or [] if args.changed_from: with open(args.changed_from, 'r') as changes_fd: paths += changes_fd.read().splitlines() elif args.changed: paths = detect_changes_local(args) else: return None # change detection not enabled display.info('Detected changes in %d file(s).' % len(paths)) for path in paths: display.info(path, verbosity=1) return paths
def command_sanity_validate_modules(args, targets): """ :type args: SanityConfig :type targets: SanityTargets """ env = ansible_environment(args) paths = [ deepest_path(i.path, 'lib/ansible/modules/') for i in targets.include_external ] paths = sorted(set(p for p in paths if p)) if not paths: display.info('No tests applicable.', verbosity=1) return cmd = ['test/sanity/validate-modules/validate-modules'] + paths with open('test/sanity/validate-modules/skip.txt', 'r') as skip_fd: skip_paths = skip_fd.read().splitlines() skip_paths += [e.path for e in targets.exclude_external] if skip_paths: cmd += ['--exclude', '^(%s)' % '|'.join(skip_paths)] if is_shippable(): cmd.extend(['--base-branch', os.environ['BASE_BRANCH']]) else: display.warning( "Cannot perform module comparison against the base branch when running locally" ) run_command(args, cmd, env=env)
def detect_changes(args): """ :type args: TestConfig :rtype: list[str] | None """ if args.changed and is_shippable(): display.info('Shippable detected, collecting parameters from environment.') paths = detect_changes_shippable(args) elif args.changed_from or args.changed_path: paths = args.changed_path or [] if args.changed_from: with open(args.changed_from, 'r') as changes_fd: paths += changes_fd.read().splitlines() elif args.changed: paths = detect_changes_local(args) else: return None # change detection not enabled if paths is None: return None # act as though change detection not enabled, do not filter targets display.info('Detected changes in %d file(s).' % len(paths)) for path in paths: display.info(path, verbosity=1) return paths
def filter(self, targets, exclude): """Filter out the cloud tests when the necessary config and resources are not available. :type targets: tuple[TestTarget] :type exclude: list[str] """ if self.vmware_test_platform is None or 'govcsim': docker = find_executable('docker', required=False) if docker: return skip = 'cloud/%s/' % self.platform skipped = [target.name for target in targets if skip in target.aliases] if skipped: exclude.append(skip) display.warning('Excluding tests marked "%s" which require the "docker" command: %s' % (skip.rstrip('/'), ', '.join(skipped))) else: if os.path.isfile(self.config_static_path): return aci = self._create_ansible_core_ci() if os.path.isfile(aci.ci_key): return if is_shippable(): return super(VcenterProvider, self).filter(targets, exclude)
def cleanup(self): """Clean up the cloud resource and any temporary configuration files after tests complete.""" if self.container_name: if is_shippable(): docker_rm(self.args, self.container_name) elif not self.args.explain: display.notice('Remember to run `docker rm -f %s` when finished testing.' % self.container_name) super(CsCloudProvider, self).cleanup()
def __init__(self): """Initialize metadata.""" self.changes = {} # type: dict [str, tuple[tuple[int, int]] self.cloud_config = None # type: dict [str, str] if is_shippable(): self.ci_provider = 'shippable' else: self.ci_provider = ''
def __init__(self): """Initialize metadata.""" self.changes = {} # type: dict [str, tuple[tuple[int, int]] self.cloud_config = None # type: dict [str, str] self.instance_config = None # type: list[dict[str, str]] self.change_description = None # type: ChangeDescription if is_shippable(): self.ci_provider = 'shippable' else: self.ci_provider = ''
def start(self): """Start instance.""" if self.started: display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) return None if is_shippable(): return self.start_shippable() return self.start_remote()
def __init__(self): """Initialize metadata.""" self.changes = {} # type: t.Dict[str, t.Tuple[t.Tuple[int, int]]] self.cloud_config = None # type: t.Optional[t.Dict[str, str]] self.instance_config = None # type: t.Optional[t.List[t.Dict[str, str]]] self.change_description = None # type: t.Optional[ChangeDescription] if is_shippable(): self.ci_provider = 'shippable' else: self.ci_provider = ''
def start(self): """Start instance.""" if self.started: display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) return if is_shippable(): return self.start_shippable() return self.start_remote()
def _generate_resource_prefix(): """ :rtype: str """ if is_shippable(): return 'shippable-%s-%s' % ( os.environ['SHIPPABLE_BUILD_NUMBER'], os.environ['SHIPPABLE_JOB_NUMBER'], ) node = re.sub(r'[^a-zA-Z0-9]+', '-', platform.node().split('.')[0]).lower() return 'ansible-test-%s-%d' % (node, random.randint(10000000, 99999999))
def filter(self, targets, exclude): """Filter out the cloud tests when the necessary config and resources are not available. :type targets: tuple[TestTarget] :type exclude: list[str] """ if os.path.isfile(self.config_static_path): return aci = get_tower_aci(self.args) if os.path.isfile(aci.ci_key): return if is_shippable(): return super(TowerCloudProvider, self).filter(targets, exclude)
def __init__(self, args, command): """ :type args: any :type command: str """ self.command = command self.color = args.color # type: bool self.explain = args.explain # type: bool self.verbosity = args.verbosity # type: int self.debug = args.debug # type: bool self.truncate = args.truncate # type: int self.redact = args.redact # type: bool if is_shippable(): self.redact = True self.cache = {}
def __init__(self, args): """ :type args: any """ super(SanityConfig, self).__init__(args, 'sanity') self.test = args.test # type: list [str] self.skip_test = args.skip_test # type: list [str] self.list_tests = args.list_tests # type: bool if args.base_branch: self.base_branch = args.base_branch # str elif is_shippable(): self.base_branch = os.environ.get('BASE_BRANCH', '') # str if self.base_branch: self.base_branch = 'origin/%s' % self.base_branch else: self.base_branch = ''
def __init__(self, args): """ :type args: any """ super(SanityConfig, self).__init__(args, 'sanity') self.test = args.test # type: list [str] self.skip_test = args.skip_test # type: list [str] self.list_tests = args.list_tests # type: bool if args.base_branch: self.base_branch = args.base_branch # str elif is_shippable(): try: self.base_branch = os.environ['BASE_BRANCH'] # str except KeyError as ex: raise MissingEnvironmentVariable(name=ex.args[0]) else: self.base_branch = ''
def filter(self, targets, exclude): """Filter out the cloud tests when the necessary config and resources are not available. :type targets: tuple[TestTarget] :type exclude: list[str] """ if os.path.isfile(self.config_static_path): return aci = self._create_ansible_core_ci() if os.path.isfile(aci.ci_key): return if os.path.isfile(self.SHERLOCK_CONFIG_PATH): return if is_shippable(): return super(AzureCloudProvider, self).filter(targets, exclude)
def start(self): """Start instance.""" if is_shippable(): self.start_shippable() else: self.start_remote()
def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None, provider=None): """ :type args: EnvironmentConfig :type platform: str :type version: str :type stage: str :type persist: bool :type load: bool :type name: str """ self.args = args self.platform = platform self.version = version self.stage = stage self.client = HttpClient(args) self.connection = None self.instance_id = None self.endpoint = None self.max_threshold = 1 self.name = name if name else '%s-%s' % (self.platform, self.version) self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') self.resource = 'jobs' # Assign each supported platform to one provider. # This is used to determine the provider from the platform when no provider is specified. providers = dict( aws=( 'aws', 'windows', 'freebsd', 'vyos', 'junos', 'ios', 'tower', 'rhel', ), azure=('azure', ), parallels=('osx', ), ) if provider: # override default provider selection (not all combinations are valid) self.provider = provider else: for candidate in providers: if platform in providers[candidate]: # assign default provider based on platform self.provider = candidate break for candidate in providers: if '%s/%s' % (platform, version) in providers[candidate]: # assign default provider based on platform and version self.provider = candidate break self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage)) if self.provider in ('aws', 'azure'): if self.provider != 'aws': self.resource = self.provider if args.remote_aws_region: # permit command-line override of region selection region = args.remote_aws_region # use a dedicated CI key when overriding the region selection self.ci_key += '.%s' % args.remote_aws_region elif is_shippable(): # split Shippable jobs across multiple regions to maximize use of launch credits if self.platform == 'windows': region = 'us-east-2' else: region = 'us-east-1' else: # send all non-Shippable jobs to us-east-1 to reduce api key maintenance region = 'us-east-1' self.path = "%s-%s" % (self.path, region) self.endpoints = (AWS_ENDPOINTS[region], ) self.ssh_key = SshKey(args) if self.platform == 'windows': self.port = 5986 else: self.port = 22 elif self.provider == 'parallels': self.endpoints = self._get_parallels_endpoints() self.max_threshold = 6 self.ssh_key = SshKey(args) self.port = None else: raise ApplicationError('Unsupported platform: %s' % platform) if persist and load and self._load(): try: display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.connection = self.get(always_raise_on=[404]) display.info('Loaded existing %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) except HttpError as ex: if ex.status != 404: raise self._clear() display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.instance_id = None self.endpoint = None elif not persist: self.instance_id = None self.endpoint = None self._clear() if self.instance_id: self.started = True else: self.started = False self.instance_id = str(uuid.uuid4()) self.endpoint = None display.sensitive.add(self.instance_id)
def __init__(self, args, platform, version, stage='prod', persist=True, name=None): """ :type args: EnvironmentConfig :type platform: str :type version: str :type stage: str :type persist: bool :type name: str """ self.args = args self.platform = platform self.version = version self.stage = stage self.client = HttpClient(args) self.connection = None self.instance_id = None self.endpoint = None self.max_threshold = 1 self.name = name if name else '%s-%s' % (self.platform, self.version) self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') aws_platforms = ( 'aws', 'azure', 'windows', 'freebsd', 'rhel', 'vyos', 'junos', 'ios', ) osx_platforms = ('osx', ) if self.platform in aws_platforms: if args.remote_aws_region: # permit command-line override of region selection region = args.remote_aws_region # use a dedicated CI key when overriding the region selection self.ci_key += '.%s' % args.remote_aws_region elif is_shippable(): # split Shippable jobs across multiple regions to maximize use of launch credits if self.platform == 'windows': region = 'us-east-2' else: region = 'us-east-1' else: # send all non-Shippable jobs to us-east-1 to reduce api key maintenance region = 'us-east-1' self.endpoints = AWS_ENDPOINTS[region], if self.platform == 'windows': self.ssh_key = None self.port = 5986 else: self.ssh_key = SshKey(args) self.port = 22 elif self.platform in osx_platforms: self.endpoints = self._get_parallels_endpoints() self.max_threshold = 6 self.ssh_key = SshKey(args) self.port = None else: raise ApplicationError('Unsupported platform: %s' % platform) self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage)) if persist and self._load(): try: display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.connection = self.get(always_raise_on=[404]) display.info('Loaded existing %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) except HttpError as ex: if ex.status != 404: raise self._clear() display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.instance_id = None self.endpoint = None else: self.instance_id = None self.endpoint = None self._clear() if self.instance_id: self.started = True else: self.started = False self.instance_id = str(uuid.uuid4()) self.endpoint = None
def start(self): """Start instance.""" if is_shippable(): return self.start_shippable() return self.start_remote()
def __init__(self, args, platform, version, stage='prod', persist=True, name=None): """ :type args: EnvironmentConfig :type platform: str :type version: str :type stage: str :type persist: bool :type name: str """ self.args = args self.platform = platform self.version = version self.stage = stage self.client = HttpClient(args) self.connection = None self.instance_id = None self.name = name if name else '%s-%s' % (self.platform, self.version) self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') aws_platforms = ( 'aws', 'azure', 'windows', 'freebsd', 'rhel', 'vyos', 'junos', 'ios', ) osx_platforms = ( 'osx', ) if self.platform in aws_platforms: if args.remote_aws_region: # permit command-line override of region selection region = args.remote_aws_region # use a dedicated CI key when overriding the region selection self.ci_key += '.%s' % args.remote_aws_region elif is_shippable(): # split Shippable jobs across multiple regions to maximize use of launch credits if self.platform == 'windows': region = 'us-east-2' else: region = 'us-east-1' else: # send all non-Shippable jobs to us-east-1 to reduce api key maintenance region = 'us-east-1' self.endpoint = AWS_ENDPOINTS[region] if self.platform == 'windows': self.ssh_key = None self.port = 5986 else: self.ssh_key = SshKey(args) self.port = 22 elif self.platform in osx_platforms: self.endpoint = 'https://osx.testing.ansible.com' self.ssh_key = SshKey(args) self.port = None else: raise ApplicationError('Unsupported platform: %s' % platform) self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage)) if persist and self._load(): try: display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.connection = self.get(always_raise_on=[404]) display.info('Loaded existing %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) except HttpError as ex: if ex.status != 404: raise self._clear() display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.instance_id = None else: self.instance_id = None self._clear() if self.instance_id: self.started = True else: self.started = False self.instance_id = str(uuid.uuid4())
def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None, provider=None): """ :type args: EnvironmentConfig :type platform: str :type version: str :type stage: str :type persist: bool :type load: bool :type name: str """ self.args = args self.platform = platform self.version = version self.stage = stage self.client = HttpClient(args) self.connection = None self.instance_id = None self.endpoint = None self.max_threshold = 1 self.name = name if name else '%s-%s' % (self.platform, self.version) self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') self.resource = 'jobs' # Assign each supported platform to one provider. # This is used to determine the provider from the platform when no provider is specified. providers = dict( aws=( 'aws', 'windows', 'freebsd', 'vyos', 'junos', 'ios', 'tower', ), azure=( 'azure', 'rhel', 'windows/2012', 'windows/2012-R2', 'windows/2016', ), parallels=( 'osx', ), ) if provider: # override default provider selection (not all combinations are valid) self.provider = provider else: for candidate in providers: if platform in providers[candidate]: # assign default provider based on platform self.provider = candidate break for candidate in providers: if '%s/%s' % (platform, version) in providers[candidate]: # assign default provider based on platform and version self.provider = candidate break if self.provider in ('aws', 'azure'): if self.provider != 'aws': self.resource = self.provider if args.remote_aws_region: # permit command-line override of region selection region = args.remote_aws_region # use a dedicated CI key when overriding the region selection self.ci_key += '.%s' % args.remote_aws_region elif is_shippable(): # split Shippable jobs across multiple regions to maximize use of launch credits if self.platform == 'windows': region = 'us-east-2' else: region = 'us-east-1' else: # send all non-Shippable jobs to us-east-1 to reduce api key maintenance region = 'us-east-1' self.endpoints = AWS_ENDPOINTS[region], if self.platform == 'windows': self.ssh_key = None self.port = 5986 else: self.ssh_key = SshKey(args) self.port = 22 elif self.provider == 'parallels': self.endpoints = self._get_parallels_endpoints() self.max_threshold = 6 self.ssh_key = SshKey(args) self.port = None else: raise ApplicationError('Unsupported platform: %s' % platform) self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage)) if persist and load and self._load(): try: display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.connection = self.get(always_raise_on=[404]) display.info('Loaded existing %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) except HttpError as ex: if ex.status != 404: raise self._clear() display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) self.instance_id = None self.endpoint = None elif not persist: self.instance_id = None self.endpoint = None self._clear() if self.instance_id: self.started = True else: self.started = False self.instance_id = str(uuid.uuid4()) self.endpoint = None display.sensitive.add(self.instance_id)