def fetch_desired_state(clusters, vault_input_path, settings): desired_state = [] error = False secret_reader = SecretReader(settings=settings) for cluster_info in clusters: cluster = cluster_info['name'] auth = cluster_info['auth'] service = auth['service'] if service != 'github-org-team': # currently not supported continue org = auth['org'] team = auth['team'] secret_path = f'{vault_input_path}/{QONTRACT_INTEGRATION}/' + \ f'{service}/{org}/{team}' secret = {'path': secret_path} try: oauth_data = secret_reader.read_all(secret) client_id = oauth_data['client-id'] client_secret = oauth_data['client-secret'] except Exception: logging.error(f"unable to read secret in path {secret['path']}") error = True continue item = { 'cluster': cluster, 'name': f'github-{org}', 'client_id': client_id, 'client_secret': client_secret, 'teams': [f'{org}/{team}'] } desired_state.append(item) return desired_state, error
def fetch_desired_state(clusters, vault_input_path, settings): desired_state = [] error = False secret_reader = SecretReader(settings=settings) for cluster_info in clusters: cluster = cluster_info["name"] auth = cluster_info["auth"] service = auth["service"] if service != "github-org-team": # currently not supported continue org = auth["org"] team = auth["team"] secret_path = ( f"{vault_input_path}/{QONTRACT_INTEGRATION}/" + f"{service}/{org}/{team}" ) secret = {"path": secret_path} try: oauth_data = secret_reader.read_all(secret) client_id = oauth_data["client-id"] client_secret = oauth_data["client-secret"] except Exception: logging.error(f"unable to read secret in path {secret['path']}") error = True continue item = { "cluster": cluster, "name": f"github-{org}", "client_id": client_id, "client_secret": client_secret, "teams": [f"{org}/{team}"], } desired_state.append(item) return desired_state, error
def __init__(self, dry_run, thread_pool_size): self.dry_run = dry_run self.thread_pool_size = thread_pool_size self.settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=self.settings) secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET}) self.dashdotdb_url = secret_content['url'] self.dashdotdb_user = secret_content['username'] self.dashdotdb_pass = secret_content['password'] self.logmarker = "DDDB_DVO:"
def get_smtp_config(path, settings): config = {} required_keys = ('password', 'port', 'require_tls', 'server', 'username') secret_reader = SecretReader(settings=settings) data = secret_reader.read_all({'path': path}) try: for k in required_keys: config[k] = data[k] except KeyError as e: raise Exception(f'Missing expected SMTP config ' f'key in vault secret: {e}') return config
def aws_creds(ctx, account_name): settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) accounts = queries.get_aws_accounts(name=account_name) if not accounts: print(f"{account_name} not found.") sys.exit(1) account = accounts[0] secret = secret_reader.read_all(account['automationToken']) print(f"export AWS_REGION={account['resourcesDefaultRegion']}") print(f"export AWS_ACCESS_KEY_ID={secret['aws_access_key_id']}") print(f"export AWS_SECRET_ACCESS_KEY={secret['aws_secret_access_key']}")
def get_smtp_config(settings): config = {} required_keys = ("password", "port", "require_tls", "server", "username") secret_reader = SecretReader(settings=settings) data = secret_reader.read_all(settings["smtp"]["credentials"]) try: for k in required_keys: config[k] = data[k] except KeyError as e: raise Exception(f"Missing expected SMTP config " f"key in vault secret: {e}") return config
def run(dry_run: bool, enable_deletion: bool): settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) creds_path = get_config().get("dyn", {}).get("secrets_path", None) if not creds_path: raise ConfigNotFound("Dyn config missing from config file") creds = secret_reader.read_all({"path": creds_path}) dyn_session.DynectSession(creds["customer"], creds["dyn_id"], creds["password"]) desired = fetch_desired_state() current = fetch_current_state() process_tds( current["tds"], desired["tds"], dry_run=dry_run, enable_deletion=enable_deletion )
def run(dry_run: bool, enable_deletion: bool): settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) creds_path = get_config().get('dyn', {}).get('secrets_path', None) if not creds_path: raise ConfigNotFound('Dyn config missing from config file') creds = secret_reader.read_all({'path': creds_path}) dyn_session.DynectSession(creds['customer'], creds['dyn_id'], creds['password']) desired = fetch_desired_state() current = fetch_current_state() process_tds(current['tds'], desired['tds'], dry_run=dry_run, enable_deletion=enable_deletion)
def run(dry_run): instance = queries.get_gitlab_instance() settings = queries.get_app_interface_settings() gl = GitLabApi(instance, settings=settings) secret_reader = SecretReader(settings=settings) # Jira repos = queries.get_repos_gitlab_jira(server=gl.server) for repo in repos: skip = False repo_url = repo['url'] project = gl.get_project(repo_url=repo_url) services = project.services current_jira = services.get('jira') desired_jira = repo['jira'] desired_jira_url = desired_jira['serverUrl'] desired_jira_crdentials = \ secret_reader.read_all(desired_jira['token']) if current_jira.active: properties = current_jira.properties desired_jira_username = desired_jira_crdentials['username'] if properties['url'] == desired_jira_url \ and properties['username'] == desired_jira_username: skip = True if skip: continue logging.info(['update_jira', repo_url, desired_jira_url]) if not dry_run: new_data = { 'active': True, 'url': desired_jira_url, 'username': desired_jira_crdentials['username'], 'password': desired_jira_crdentials['password'], 'commit_events': True, 'merge_requests_events': True, 'comment_on_event_enabled': False } services.update('jira', new_data=new_data)
def run(dry_run): instance = queries.get_gitlab_instance() settings = queries.get_app_interface_settings() gl = GitLabApi(instance, settings=settings) secret_reader = SecretReader(settings=settings) # Jira repos = queries.get_repos_gitlab_jira(server=gl.server) for repo in repos: skip = False repo_url = repo["url"] services = get_repo_services(gl, repo_url) current_jira = services.get("jira") desired_jira = repo["jira"] desired_jira_url = desired_jira["serverUrl"] desired_jira_crdentials = secret_reader.read_all(desired_jira["token"]) if current_jira.active: properties = current_jira.properties desired_jira_username = desired_jira_crdentials["username"] if (properties["url"] == desired_jira_url and properties["username"] == desired_jira_username): skip = True if skip: continue logging.info(["update_jira", repo_url, desired_jira_url]) if not dry_run: new_data = { "active": True, "url": desired_jira_url, "username": desired_jira_crdentials["username"], "password": desired_jira_crdentials["password"], "commit_events": True, "merge_requests_events": True, "comment_on_event_enabled": False, } services.update("jira", new_data=new_data)
def get_quay_api_store(): """ Returns a dictionary with a key for each Quay organization managed in app-interface. Each key contains an initiated QuayApi instance. """ quay_orgs = queries.get_quay_orgs() settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) store = {} for org_data in quay_orgs: instance_name = org_data['instance']['name'] org_name = org_data['name'] org_key = OrgKey(instance_name, org_name) base_url = org_data['instance']['url'] token = secret_reader.read(org_data['automationToken']) if org_data.get('mirror'): mirror = OrgKey(org_data['mirror']['instance']['name'], org_data['mirror']['name']) else: mirror = None if org_data.get('pushCredentials'): push_token = secret_reader.read_all(org_data['pushCredentials']) else: push_token = None store[org_key] = { 'url': base_url, 'api': QuayApi(token, org_name, base_url=base_url), 'push_token': push_token, 'teams': org_data.get('managedTeams'), 'managedRepos': org_data.get('managedRepos'), 'mirror': mirror, } return store
class JiraClient: """Wrapper around Jira client""" def __init__(self, jira_board, settings=None): self.secret_reader = SecretReader(settings=settings) self.project = jira_board['name'] jira_server = jira_board['server'] self.server = jira_server['serverUrl'] token = jira_server['token'] basic_auth = self.get_basic_auth(token) self.jira = JIRA(self.server, basic_auth=basic_auth) def get_basic_auth(self, token): required_keys = ['username', 'password'] secret = self.secret_reader.read_all(token) ok = all(elem in secret.keys() for elem in required_keys) if not ok: raise KeyError('[{}] secret is missing required keys'.format( self.project)) return (secret['username'], secret['password']) def get_issues(self, fields=None): block_size = 100 block_num = 0 all_issues = [] jql = 'project={}'.format(self.project) kwargs = {} if fields: kwargs['fields'] = ','.join(fields) while True: index = block_num * block_size issues = self.jira.search_issues(jql, index, block_size, **kwargs) all_issues.extend(issues) if len(issues) < block_size: break block_num += 1 return all_issues
def get_apps_data(date, month_delta=1, thread_pool_size=10): apps = queries.get_apps() saas_files = queries.get_saas_files() jjb, _ = init_jjb() jenkins_map = jenkins_base.get_jenkins_map() time_limit = date - relativedelta(months=month_delta) timestamp_limit = \ int(time_limit.replace(tzinfo=timezone.utc).timestamp()) settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET}) dashdotdb_url = secret_content['url'] dashdotdb_user = secret_content['username'] dashdotdb_pass = secret_content['password'] auth = (dashdotdb_user, dashdotdb_pass) vuln_metrics = requests.get( f'{dashdotdb_url}/api/v1/imagemanifestvuln/metrics', auth=auth).text validt_metrics = requests.get( f'{dashdotdb_url}/api/v1/deploymentvalidation/metrics', auth=auth).text slo_metrics = requests.get( f'{dashdotdb_url}/api/v1/serviceslometrics/metrics', auth=auth).text namespaces = queries.get_namespaces() build_jobs = jjb.get_all_jobs(job_types=['build']) jobs_to_get = build_jobs.copy() saas_deploy_jobs = [] for saas_file in saas_files: saas_file_name = saas_file['name'] for template in saas_file["resourceTemplates"]: for target in template["targets"]: job = {} job['env'] = target["namespace"]["environment"]["name"] job['app'] = target["namespace"]["app"]["name"] job['cluster'] = target['namespace']['cluster']['name'] job['namespace'] = target['namespace']['name'] job['name'] = get_openshift_saas_deploy_job_name( saas_file_name, job['env'], settings) job['saas_file_name'] = saas_file_name job['instance'] = saas_file["instance"]["name"] saas_deploy_jobs.append(job) if job['instance'] not in jobs_to_get: jobs_to_get[job['instance']] = [job] else: jobs_to_get[job['instance']].append(job) job_history = get_build_history_pool(jenkins_map, jobs_to_get, timestamp_limit, thread_pool_size) for app in apps: if not app['codeComponents']: continue app_name = app['name'] logging.info(f"collecting post-deploy jobs " f"information for {app_name}") post_deploy_jobs = {} for saas_file in saas_files: if saas_file['app']['name'] != app_name: continue resource_types = saas_file['managedResourceTypes'] # Only jobs of these types are expected to have a # further post-deploy job if not any([ 'Deployment' in resource_types, 'DeploymentConfig' not in resource_types ]): continue for resource_template in saas_file['resourceTemplates']: for target in resource_template['targets']: cluster = target['namespace']['cluster']['name'] namespace = target['namespace']['name'] post_deploy_jobs[cluster] = {} post_deploy_jobs[cluster][namespace] = False for saas_file in saas_files: if saas_file['app']['name'] != app_name: continue resource_types = saas_file['managedResourceTypes'] if 'Job' not in resource_types: continue for resource_template in saas_file['resourceTemplates']: for target in resource_template['targets']: cluster = target['namespace']['cluster']['name'] namespace = target['namespace']['name'] # This block skips the check if the cluster/namespace # has no Deployment/DeploymentConfig job associated. if cluster not in post_deploy_jobs: continue if namespace not in post_deploy_jobs[cluster]: continue # Post-deploy job must depend on a openshift-saas-deploy # job if target['upstream'] is None: continue if target['upstream'].startswith('openshift-saas-deploy-'): post_deploy_jobs[cluster][namespace] = True app['post_deploy_jobs'] = post_deploy_jobs logging.info(f"collecting promotion history for {app_name}") app["promotions"] = {} for job in saas_deploy_jobs: if job['app'] != app_name: continue if job['name'] not in job_history: continue history = job_history[job["name"]] saas_file_name = job['saas_file_name'] if saas_file_name not in app["promotions"]: app["promotions"][saas_file_name] = [{ "env": job["env"], "cluster": job["cluster"], "namespace": job["namespace"], **history }] else: app["promotions"][saas_file_name].append({ "env": job["env"], "cluster": job["cluster"], "namespace": job["namespace"], **history }) logging.info(f"collecting merge activity for {app_name}") app['merge_activity'] = {} code_repos = [ c['url'] for c in app['codeComponents'] if c['resource'] == 'upstream' ] for jobs in build_jobs.values(): for job in jobs: try: repo_url = get_repo_url(job) except KeyError: continue if repo_url not in code_repos: continue if job['name'] not in job_history: continue history = job_history[job['name']] if repo_url not in app["merge_activity"]: app["merge_activity"][repo_url] = [{ "branch": job["branch"], **history }] else: app["merge_activity"][repo_url].append({ "branch": job["branch"], **history }) logging.info(f"collecting dashdotdb information for {app_name}") app_namespaces = [] for namespace in namespaces: if namespace['app']['name'] != app['name']: continue app_namespaces.append(namespace) vuln_mx = {} validt_mx = {} slo_mx = {} for family in text_string_to_metric_families(vuln_metrics): for sample in family.samples: if sample.name == 'imagemanifestvuln_total': for app_namespace in app_namespaces: cluster = sample.labels['cluster'] if app_namespace['cluster']['name'] != cluster: continue namespace = sample.labels['namespace'] if app_namespace['name'] != namespace: continue severity = sample.labels['severity'] if cluster not in vuln_mx: vuln_mx[cluster] = {} if namespace not in vuln_mx[cluster]: vuln_mx[cluster][namespace] = {} if severity not in vuln_mx[cluster][namespace]: value = int(sample.value) vuln_mx[cluster][namespace][severity] = value for family in text_string_to_metric_families(validt_metrics): for sample in family.samples: if sample.name == 'deploymentvalidation_total': for app_namespace in app_namespaces: cluster = sample.labels['cluster'] if app_namespace['cluster']['name'] != cluster: continue namespace = sample.labels['namespace'] if app_namespace['name'] != namespace: continue validation = sample.labels['validation'] # dvo: fail == 1, pass == 0, py: true == 1, false == 0 # so: ({false|pass}, {true|fail}) status = ('Passed', 'Failed')[int(sample.labels['status'])] if cluster not in validt_mx: validt_mx[cluster] = {} if namespace not in validt_mx[cluster]: validt_mx[cluster][namespace] = {} if validation not in validt_mx[cluster][namespace]: validt_mx[cluster][namespace][validation] = {} if status not in validt_mx[cluster][namespace][ validation]: # noqa: E501 validt_mx[cluster][namespace][validation][ status] = {} # noqa: E501 value = int(sample.value) validt_mx[cluster][namespace][validation][ status] = value # noqa: E501 for family in text_string_to_metric_families(slo_metrics): for sample in family.samples: if sample.name == 'serviceslometrics': for app_namespace in app_namespaces: cluster = sample.labels['cluster'] if app_namespace['cluster']['name'] != cluster: continue namespace = sample.labels['namespace'] if app_namespace['name'] != namespace: continue slo_doc_name = sample.labels['slodoc'] slo_name = sample.labels['name'] if cluster not in slo_mx: slo_mx[cluster] = {} if namespace not in slo_mx[cluster]: slo_mx[cluster][namespace] = {} if slo_doc_name not in slo_mx[cluster][namespace]: # pylint: disable=line-too-long # noqa: E501 slo_mx[cluster][namespace][slo_doc_name] = {} if slo_name not in slo_mx[cluster][namespace][ slo_doc_name]: # noqa: E501 slo_mx[cluster][namespace][slo_doc_name][ slo_name] = { # noqa: E501 sample.labels['type']: sample.value } else: slo_mx[cluster][namespace][slo_doc_name][ slo_name].update({ # pylint: disable=line-too-long # noqa: E501 sample.labels['type']: sample.value }) app['container_vulnerabilities'] = vuln_mx app['deployment_validations'] = validt_mx app['service_slo'] = slo_mx return apps
class EcrMirror: def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance["identifier"] account = instance["account"] region = instance.get("region") self.aws_cli = AWSApi( thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True, ) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}" self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance["mirror"]["pullCredentials"] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f"{self.image_username}:{self.image_password}" def run(self): if self.error: return ecr_mirror = Image(self.ecr_uri, username=self.ecr_username, password=self.ecr_password) image = Image( self.instance["mirror"]["url"], username=self.image_username, password=self.image_password, ) LOG.debug("[checking %s -> %s]", image, ecr_mirror) for tag in image: if tag not in ecr_mirror: try: self.skopeo_cli.copy( src_image=image[tag], src_creds=self.image_auth, dst_image=ecr_mirror[tag], dest_creds=self.ecr_auth, ) except SkopeoCmdError as details: LOG.error("[%s]", details) def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]["resourcesDefaultRegion"] auth_token = f"{account}/{region}" data = self.aws_cli.auth_tokens[auth_token] auth_data = data["authorizationData"][0] token = auth_data["authorizationToken"] password = base64.b64decode(token).decode("utf-8").split(":")[1] return "AWS", password def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]["ecr"]: if repo["repositoryName"] == repository: return repo["repositoryUri"] @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if "name" not in account_info: continue if account_info["name"] != account: continue return account_info
class AWSApi: """Wrapper around AWS SDK""" def __init__(self, thread_pool_size, accounts, settings=None, init_ecr_auth_tokens=False): self.thread_pool_size = thread_pool_size self.secret_reader = SecretReader(settings=settings) self.init_sessions_and_resources(accounts) if init_ecr_auth_tokens: self.init_ecr_auth_tokens(accounts) self.init_users() self._lock = Lock() self.resource_types = \ ['s3', 'sqs', 'dynamodb', 'rds', 'rds_snapshots'] # store the app-interface accounts in a dictionary indexed by name self.accounts = {acc['name']: acc for acc in accounts} def init_sessions_and_resources(self, accounts): results = threaded.run(self.get_tf_secrets, accounts, self.thread_pool_size) self.sessions = {} self.resources = {} for account, secret in results: access_key = secret['aws_access_key_id'] secret_key = secret['aws_secret_access_key'] region_name = secret['region'] session = boto3.Session( aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, ) self.sessions[account] = session self.resources[account] = {} def get_session(self, account): return self.sessions[account] def get_tf_secrets(self, account): account_name = account['name'] automation_token = account['automationToken'] secret = self.secret_reader.read_all(automation_token) return (account_name, secret) def init_users(self): self.users = {} for account, s in self.sessions.items(): iam = s.client('iam') users = [u['UserName'] for u in iam.list_users()['Users']] self.users[account] = users def simulate_deleted_users(self, io_dir): src_integrations = ['terraform_resources', 'terraform_users'] if not os.path.exists(io_dir): return for i in src_integrations: file_path = os.path.join(io_dir, i + '.json') if not os.path.exists(file_path): continue with open(file_path, 'r') as f: deleted_users = json.load(f) for deleted_user in deleted_users: delete_from_account = deleted_user['account'] delete_user = deleted_user['user'] self.users[delete_from_account].remove(delete_user) def map_resources(self): threaded.run(self.map_resource, self.resource_types, self.thread_pool_size) def map_resource(self, resource_type): if resource_type == 's3': self.map_s3_resources() elif resource_type == 'sqs': self.map_sqs_resources() elif resource_type == 'dynamodb': self.map_dynamodb_resources() elif resource_type == 'rds': self.map_rds_resources() elif resource_type == 'rds_snapshots': self.map_rds_snapshots() elif resource_type == 'route53': self.map_route53_resources() else: raise InvalidResourceTypeError(resource_type) def map_s3_resources(self): for account, s in self.sessions.items(): s3 = s.client('s3') buckets_list = s3.list_buckets() if 'Buckets' not in buckets_list: continue buckets = [b['Name'] for b in buckets_list['Buckets']] self.set_resouces(account, 's3', buckets) buckets_without_owner = \ self.get_resources_without_owner(account, buckets) unfiltered_buckets = \ self.custom_s3_filter(account, s3, buckets_without_owner) self.set_resouces(account, 's3_no_owner', unfiltered_buckets) def map_sqs_resources(self): for account, s in self.sessions.items(): sqs = s.client('sqs') queues_list = sqs.list_queues() if 'QueueUrls' not in queues_list: continue queues = queues_list['QueueUrls'] self.set_resouces(account, 'sqs', queues) queues_without_owner = \ self.get_resources_without_owner(account, queues) unfiltered_queues = \ self.custom_sqs_filter(account, sqs, queues_without_owner) self.set_resouces(account, 'sqs_no_owner', unfiltered_queues) def map_dynamodb_resources(self): for account, s in self.sessions.items(): dynamodb = s.client('dynamodb') tables = self.paginate(dynamodb, 'list_tables', 'TableNames') self.set_resouces(account, 'dynamodb', tables) tables_without_owner = \ self.get_resources_without_owner(account, tables) unfiltered_tables = \ self.custom_dynamodb_filter( account, s, dynamodb, tables_without_owner ) self.set_resouces(account, 'dynamodb_no_owner', unfiltered_tables) def map_rds_resources(self): for account, s in self.sessions.items(): rds = s.client('rds') results = \ self.paginate(rds, 'describe_db_instances', 'DBInstances') instances = [t['DBInstanceIdentifier'] for t in results] self.set_resouces(account, 'rds', instances) instances_without_owner = \ self.get_resources_without_owner(account, instances) unfiltered_instances = \ self.custom_rds_filter(account, rds, instances_without_owner) self.set_resouces(account, 'rds_no_owner', unfiltered_instances) def map_rds_snapshots(self): self.wait_for_resource('rds') for account, s in self.sessions.items(): rds = s.client('rds') results = \ self.paginate(rds, 'describe_db_snapshots', 'DBSnapshots') snapshots = [t['DBSnapshotIdentifier'] for t in results] self.set_resouces(account, 'rds_snapshots', snapshots) snapshots_without_db = [t['DBSnapshotIdentifier'] for t in results if t['DBInstanceIdentifier'] not in self.resources[account]['rds']] unfiltered_snapshots = \ self.custom_rds_snapshot_filter(account, rds, snapshots_without_db) self.set_resouces(account, 'rds_snapshots_no_owner', unfiltered_snapshots) def map_route53_resources(self): for account, s in self.sessions.items(): client = s.client('route53') results = \ self.paginate(client, 'list_hosted_zones', 'HostedZones') zones = [z for z in results] for zone in zones: results = \ self.paginate(client, 'list_resource_record_sets', 'ResourceRecordSets', {'HostedZoneId': zone['Id']}) zone['records'] = results self.set_resouces(account, 'route53', zones) def map_ecr_resources(self): for account, s in self.sessions.items(): client = s.client('ecr') repositories = self.paginate(client=client, method='describe_repositories', key='repositories') self.set_resouces(account, 'ecr', repositories) @staticmethod def paginate(client, method, key, params={}): """ paginate returns an aggregated list of the specified key from all pages returned by executing the client's specified method.""" paginator = client.get_paginator(method) return [values for page in paginator.paginate(**params) for values in page.get(key, [])] def wait_for_resource(self, resource): """ wait_for_resource waits until the specified resource type is ready for all accounts. When we have more resource types then threads, this function will need to change to a dependency graph.""" wait = True while wait: wait = False for account in self.sessions: if self.resources[account].get(resource) is None: wait = True if wait: time.sleep(2) def set_resouces(self, account, key, value): with self._lock: self.resources[account][key] = value def get_resources_without_owner(self, account, resources): return [r for r in resources if not self.has_owner(account, r)] def has_owner(self, account, resource): has_owner = False for u in self.users[account]: if resource.lower().startswith(u.lower()): has_owner = True break if '://' in resource: if resource.split('/')[-1].startswith(u.lower()): has_owner = True break return has_owner def custom_s3_filter(self, account, s3, buckets): type = 's3 bucket' unfiltered_buckets = [] for b in buckets: try: tags = s3.get_bucket_tagging(Bucket=b) except botocore.exceptions.ClientError: tags = {} if not self.should_filter(account, type, b, tags, 'TagSet'): unfiltered_buckets.append(b) return unfiltered_buckets def custom_sqs_filter(self, account, sqs, queues): type = 'sqs queue' unfiltered_queues = [] for q in queues: tags = sqs.list_queue_tags(QueueUrl=q) if not self.should_filter(account, type, q, tags, 'Tags'): unfiltered_queues.append(q) return unfiltered_queues def custom_dynamodb_filter(self, account, session, dynamodb, tables): type = 'dynamodb table' dynamodb_resource = session.resource('dynamodb') unfiltered_tables = [] for t in tables: table_arn = dynamodb_resource.Table(t).table_arn tags = dynamodb.list_tags_of_resource(ResourceArn=table_arn) if not self.should_filter(account, type, t, tags, 'Tags'): unfiltered_tables.append(t) return unfiltered_tables def custom_rds_filter(self, account, rds, instances): type = 'rds instance' unfiltered_instances = [] for i in instances: instance = rds.describe_db_instances(DBInstanceIdentifier=i) instance_arn = instance['DBInstances'][0]['DBInstanceArn'] tags = rds.list_tags_for_resource(ResourceName=instance_arn) if not self.should_filter(account, type, i, tags, 'TagList'): unfiltered_instances.append(i) return unfiltered_instances def custom_rds_snapshot_filter(self, account, rds, snapshots): type = 'rds snapshots' unfiltered_snapshots = [] for s in snapshots: snapshot = rds.describe_db_snapshots(DBSnapshotIdentifier=s) snapshot_arn = snapshot['DBSnapshots'][0]['DBSnapshotArn'] tags = rds.list_tags_for_resource(ResourceName=snapshot_arn) if not self.should_filter(account, type, s, tags, 'TagList'): unfiltered_snapshots.append(s) return unfiltered_snapshots def should_filter(self, account, resource_type, resource_name, resource_tags, tags_key): if self.resource_has_special_name(account, resource_type, resource_name): return True if tags_key in resource_tags: tags = resource_tags[tags_key] if self.resource_has_special_tags(account, resource_type, resource_name, tags): return True return False @staticmethod def resource_has_special_name(account, type, resource): skip_msg = '[{}] skipping {} '.format(account, type) + \ '({} related) {}' ignore_names = { 'production': ['prod'], 'stage': ['stage', 'staging'], 'terraform': ['terraform', '-tf-'], } for msg, tags in ignore_names.items(): for tag in tags: if tag.lower() in resource.lower(): logging.debug(skip_msg.format(msg, resource)) return True return False def resource_has_special_tags(self, account, type, resource, tags): skip_msg = '[{}] skipping {} '.format(account, type) + \ '({}={}) {}' ignore_tags = { 'ENV': ['prod', 'stage', 'staging'], 'environment': ['prod', 'stage', 'staging'], 'owner': ['app-sre'], 'managed_by_integration': [ 'terraform_resources', 'terraform_users' ], 'aws_gc_hands_off': ['true'], } for tag, ignore_values in ignore_tags.items(): for ignore_value in ignore_values: value = self.get_tag_value(tags, tag) if ignore_value.lower() in value.lower(): logging.debug(skip_msg.format(tag, value, resource)) return True return False @staticmethod def get_tag_value(tags, tag): if isinstance(tags, dict): return tags.get(tag, '') elif isinstance(tags, list): for t in tags: if t['Key'] == tag: return t['Value'] return '' def delete_resources_without_owner(self, dry_run): for account, s in self.sessions.items(): for rt in self.resource_types: for r in self.resources[account].get(rt + '_no_owner', []): logging.info(['delete_resource', account, rt, r]) if not dry_run: self.delete_resource(s, rt, r) def delete_resource(self, session, resource_type, resource_name): if resource_type == 's3': resource = session.resource(resource_type) self.delete_bucket(resource, resource_name) elif resource_type == 'sqs': client = session.client(resource_type) self.delete_queue(client, resource_name) elif resource_type == 'dynamodb': resource = session.resource(resource_type) self.delete_table(resource, resource_name) elif resource_type == 'rds': client = session.client(resource_type) self.delete_instance(client, resource_name) elif resource_type == 'rds_snapshots': client = session.client(resource_type) self.delete_snapshot(client, resource_name) else: raise InvalidResourceTypeError(resource_type) @staticmethod def delete_bucket(s3, bucket_name): bucket = s3.Bucket(bucket_name) for key in bucket.objects.all(): key.delete() bucket.delete() @staticmethod def delete_queue(sqs, queue_url): sqs.delete_queue(QueueUrl=queue_url) @staticmethod def delete_table(dynamodb, table_name): table = dynamodb.Table(table_name) table.delete() @staticmethod def delete_instance(rds, instance_name): rds.delete_db_instance( DBInstanceIdentifier=instance_name, SkipFinalSnapshot=True, DeleteAutomatedBackups=True ) @staticmethod def delete_snapshot(rds, snapshot_identifier): rds.delete_db_snapshot( DBSnapshotIdentifier=snapshot_identifier ) @staticmethod def determine_key_type(iam, user): tags = iam.list_user_tags(UserName=user)['Tags'] managed_by_integration_tag = \ [t['Value'] for t in tags if t['Key'] == 'managed_by_integration'] # if this key belongs to a user without tags, i.e. not # managed by an integration, this key is probably created # manually. disable it to leave a trace if not managed_by_integration_tag: return 'unmanaged' # if this key belongs to a user created by the # 'terraform-users' integration, we just delete the key if managed_by_integration_tag[0] == 'terraform_users': return 'user' # if this key belongs to a user created by the # 'terraform-resources' integration, we remove # the key from terraform state and let it create # a new one on its own if managed_by_integration_tag[0] == 'terraform_resources': return 'service_account' huh = 'unrecognized managed_by_integration tag: {}'.format( managed_by_integration_tag[0]) raise InvalidResourceTypeError(huh) def delete_keys(self, dry_run, keys_to_delete, working_dirs, disable_service_account_keys): error = False users_keys = self.get_users_keys() for account, s in self.sessions.items(): iam = s.client('iam') keys = keys_to_delete.get(account, []) for key in keys: user_and_user_keys = [(user, user_keys) for user, user_keys in users_keys[account].items() if key in user_keys] if not user_and_user_keys: continue # unpack single item from sequence # since only a single user can have a given key [user_and_user_keys] = user_and_user_keys user = user_and_user_keys[0] user_keys = user_and_user_keys[1] key_type = self.determine_key_type(iam, user) key_status = self.get_user_key_status(iam, user, key) if key_type == 'unmanaged' and key_status == 'Active': logging.info(['disable_key', account, user, key]) if not dry_run: iam.update_access_key( UserName=user, AccessKeyId=key, Status='Inactive' ) elif key_type == 'user': logging.info(['delete_key', account, user, key]) if not dry_run: iam.delete_access_key( UserName=user, AccessKeyId=key ) elif key_type == 'service_account': # if key is disabled - delete it # this will happen after terraform-resources ran, # provisioned a new key, updated the output Secret, # recycled the pods and disabled the key. if key_status == 'Inactive': logging.info(['delete_inactive_key', account, user, key]) if not dry_run: iam.delete_access_key( UserName=user, AccessKeyId=key ) continue # if key is active and it is the only one - # remove it from terraform state. terraform-resources # will provision a new one. # may be a race condition here. TODO: check it if len(user_keys) == 1: logging.info(['remove_from_state', account, user, key]) if not dry_run: terraform.state_rm_access_key( working_dirs, account, user ) # if user has 2 keys and we remove the key from # terraform state, terraform-resources will not # be able to provision a new key - limbo. # this state should happen when terraform-resources # is running, provisioned a new key, # but did not disable the old key yet. if len(user_keys) == 2: # if true, this is a call made by terraform-resources # itself. disable the key and proceed. the key will be # deleted in a following iteration of aws-iam-keys. if disable_service_account_keys: logging.info(['disable_key', account, user, key]) if not dry_run: iam.update_access_key( UserName=user, AccessKeyId=key, Status='Inactive' ) else: msg = \ 'user {} has 2 keys, skipping to avoid error' logging.error(msg.format(user)) error = True return error def get_users_keys(self): users_keys = {} for account, s in self.sessions.items(): iam = s.client('iam') users_keys[account] = {user: self.get_user_keys(iam, user) for user in self.users[account]} return users_keys @staticmethod def get_user_keys(iam, user): key_list = iam.list_access_keys(UserName=user)['AccessKeyMetadata'] return [uk['AccessKeyId'] for uk in key_list] @staticmethod def get_user_key_status(iam, user, key): key_list = iam.list_access_keys(UserName=user)['AccessKeyMetadata'] return [k['Status'] for k in key_list if k['AccessKeyId'] == key][0] def get_support_cases(self): all_support_cases = {} for account, s in self.sessions.items(): if not self.accounts[account].get('premiumSupport'): continue try: support = s.client('support') support_cases = support.describe_cases( includeResolvedCases=True, includeCommunications=True )['cases'] all_support_cases[account] = support_cases except Exception as e: msg = '[{}] error getting support cases. details: {}' logging.error(msg.format(account, str(e))) return all_support_cases def init_ecr_auth_tokens(self, accounts): accounts_with_ecr = [a for a in accounts if a.get('ecrs')] if not accounts_with_ecr: return auth_tokens = {} results = threaded.run(self.get_tf_secrets, accounts_with_ecr, self.thread_pool_size) account_secrets = {account: secret for account, secret in results} for account in accounts_with_ecr: account_name = account['name'] account_secret = account_secrets[account_name] access_key = account_secret['aws_access_key_id'] secret_key = account_secret['aws_secret_access_key'] ecrs = account['ecrs'] for ecr in ecrs: region_name = ecr['region'] session = boto3.Session( aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, ) client = session.client('ecr') token = client.get_authorization_token() auth_tokens[f"{account_name}/{region_name}"] = token self.auth_tokens = auth_tokens def get_cluster_vpc_id(self, account, route_tables=False): """ Returns a cluster VPC ID. :param account: a dictionary containing the following keys: - name - name of the AWS account - assume_role - role to assume to get access to the cluster's AWS account - assume_region - region in which to operate - assume_cidr - CIDR block of the cluster to use to find the matching VPC """ required_keys = \ ['name', 'assume_role', 'assume_region', 'assume_cidr'] ok = all(elem in account.keys() for elem in required_keys) if not ok: account_name = account.get('name') raise KeyError( '[{}] account is missing required keys'.format(account_name)) session = self.get_session(account['name']) sts = session.client('sts') role_arn = account['assume_role'] if not role_arn: raise KeyError( 'Could not find Role ARN. This is likely caused ' 'due to a missing awsInfrastructureAccess section.' ) role_name = role_arn.split('/')[1] response = sts.assume_role( RoleArn=role_arn, RoleSessionName=role_name ) credentials = response['Credentials'] assumed_session = boto3.Session( aws_access_key_id=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_session_token=credentials['SessionToken'], region_name=account['assume_region'] ) assumed_ec2 = assumed_session.client('ec2') vpcs = assumed_ec2.describe_vpcs() vpc_id = None for vpc in vpcs.get('Vpcs'): if vpc['CidrBlock'] == account['assume_cidr']: vpc_id = vpc['VpcId'] break route_table_ids = None if route_tables and vpc_id: vpc_route_tables = assumed_ec2.describe_route_tables( Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}] ) route_table_ids = [rt['RouteTableId'] for rt in vpc_route_tables['RouteTables']] return vpc_id, route_table_ids def get_vpcs_details(self, account, tags=None, route_tables=False): results = [] session = self.get_session(account['name']) ec2 = session.client('ec2') regions = [r['RegionName'] for r in ec2.describe_regions()['Regions']] for region_name in regions: ec2 = session.client('ec2', region_name=region_name) vpcs = ec2.describe_vpcs( Filters=[ {'Name': f'tag:{k}', 'Values': [v]} for k, v in tags.items() ] ) for vpc in vpcs.get('Vpcs'): vpc_id = vpc['VpcId'] cidr_block = vpc['CidrBlock'] route_table_ids = None if route_tables: vpc_route_tables = ec2.describe_route_tables( Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}] ) route_table_ids = [rt['RouteTableId'] for rt in vpc_route_tables['RouteTables']] item = { 'vpc_id': vpc_id, 'region': region_name, 'cidr_block': cidr_block, 'route_table_ids': route_table_ids, } results.append(item) return results def get_tgws_details(self, account, region_name, tags=None, route_tables=False): results = [] session = self.get_session(account['name']) ec2 = session.client('ec2', region_name=region_name) tgws = ec2.describe_transit_gateways( Filters=[ {'Name': f'tag:{k}', 'Values': [v]} for k, v in tags.items() ] ) for tgw in tgws.get('TransitGateways'): tgw_id = tgw['TransitGatewayId'] tgw_arn = tgw['TransitGatewayArn'] item = { 'tgw_id': tgw_id, 'tgw_arn': tgw_arn, 'region': region_name, } results.append(item) return results def get_route53_zones(self): """ Return a list of (str, dict) representing Route53 DNS zones per account :return: route53 dns zones per account :rtype: list of (str, dict) """ return { account: self.resources.get(account, {}).get('route53', []) for account, _ in self.sessions.items() } def create_route53_zone(self, account_name, zone_name): """ Create a Route53 DNS zone :param account_name: the account name to operate on :param zone_name: name of the zone to create :type account_name: str :type zone_name: str """ session = self.get_session(account_name) client = session.client('route53') try: caller_ref = f"{datetime.now()}" client.create_hosted_zone( Name=zone_name, CallerReference=caller_ref, HostedZoneConfig={ 'Comment': 'Managed by App-Interface', }, ) except client.exceptions.InvalidDomainName: logging.error(f'[{account_name}] invalid domain name {zone_name}') except client.exceptions.HostedZoneAlreadyExists: logging.error( f'[{account_name}] hosted zone already exists: {zone_name}' ) except client.exceptions.TooManyHostedZones: logging.error(f'[{account_name}] too many hosted zones in account') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}') def delete_route53_zone(self, account_name, zone_id): """ Delete a Route53 DNS zone :param account_name: the account name to operate on :param zone_id: aws zone id of the zone to delete :type account_name: str :type zone_id: str """ session = self.get_session(account_name) client = session.client('route53') try: client.delete_hosted_zone(Id=zone_id) except client.exceptions.NoSuchHostedZone: logging.error(f'[{account_name}] Error trying to delete ' f'unknown DNS zone {zone_id}') except client.exceptions.HostedZoneNotEmpty: logging.error(f'[{account_name}] Cannot delete DNS zone that ' f'is not empty {zone_id}') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}') def delete_route53_record(self, account_name, zone_id, awsdata): """ Delete a Route53 DNS zone record :param account_name: the account name to operate on :param zone_id: aws zone id of the zone to operate on :param awsdata: aws record data of the record to delete :type account_name: str :type zone_id: str :type awsdata: dict """ session = self.get_session(account_name) client = session.client('route53') try: client.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ 'Changes': [{ 'Action': 'DELETE', 'ResourceRecordSet': awsdata, }] } ) except client.exceptions.NoSuchHostedZone: logging.error(f'[{account_name}] Error trying to delete record: ' f'unknown DNS zone {zone_id}') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}') def upsert_route53_record(self, account_name, zone_id, recordset): """ Upsert a Route53 DNS zone record :param account_name: the account name to operate on :param zone_id: aws zone id of the zone to operate on :param recordset: aws record data of the record to create or update :type account_name: str :type zone_id: str :type recordset: dict """ session = self.get_session(account_name) client = session.client('route53') try: client.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ 'Changes': [{ 'Action': 'UPSERT', 'ResourceRecordSet': recordset, }] } ) except client.exceptions.NoSuchHostedZone: logging.error(f'[{account_name}] Error trying to delete record: ' f'unknown DNS zone {zone_id}') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}')
def get_apps_data(date, month_delta=1): apps = queries.get_apps() saas_files = queries.get_saas_files() jjb, _ = init_jjb() saas_jobs = jjb.get_all_jobs(job_types=['saas-deploy', 'promote-to-prod']) build_master_jobs = jjb.get_all_jobs(job_types=['build-master']) jenkins_map = jenkins_base.get_jenkins_map() time_limit = date - relativedelta(months=month_delta) timestamp_limit = \ int(time_limit.replace(tzinfo=timezone.utc).timestamp()) saas_build_history = \ get_build_history(jenkins_map, saas_jobs, timestamp_limit) build_master_build_history = \ get_build_history(jenkins_map, build_master_jobs, timestamp_limit) settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET}) dashdotdb_url = secret_content['url'] dashdotdb_user = secret_content['username'] dashdotdb_pass = secret_content['password'] metrics = requests.get(f'{dashdotdb_url}/api/v1/metrics', auth=(dashdotdb_user, dashdotdb_pass)).text namespaces = queries.get_namespaces() for app in apps: if not app['codeComponents']: continue app_name = app['name'] logging.info(f"collecting post-deploy jobs " f"information for {app_name}") post_deploy_jobs = {} for saas_file in saas_files: if saas_file['app']['name'] != app_name: continue resource_types = saas_file['managedResourceTypes'] # Only jobs of these types are expected to have a # further post-deploy job if not any([ 'Deployment' in resource_types, 'DeploymentConfig' not in resource_types ]): continue for resource_template in saas_file['resourceTemplates']: for target in resource_template['targets']: cluster = target['namespace']['cluster']['name'] namespace = target['namespace']['name'] post_deploy_jobs[cluster] = {} post_deploy_jobs[cluster][namespace] = False for saas_file in saas_files: if saas_file['app']['name'] != app_name: continue resource_types = saas_file['managedResourceTypes'] if 'Job' not in resource_types: continue for resource_template in saas_file['resourceTemplates']: for target in resource_template['targets']: cluster = target['namespace']['cluster']['name'] namespace = target['namespace']['name'] # This block skips the check if the cluster/namespace # has no Deployment/DeploymentConfig job associated. if cluster not in post_deploy_jobs: continue if namespace not in post_deploy_jobs[cluster]: continue # Post-deploy job must depend on a openshift-saas-deploy # job if target['upstream'] is None: continue if target['upstream'].startswith('openshift-saas-deploy-'): post_deploy_jobs[cluster][namespace] = True app['post_deploy_jobs'] = post_deploy_jobs logging.info(f"collecting promotions for {app_name}") app['promotions'] = {} saas_repos = [ c['url'] for c in app['codeComponents'] if c['resource'] == 'saasrepo' ] for sr in saas_repos: sr_history = saas_build_history.get(sr) if not sr_history: continue successes = [h for h in sr_history if h == 'SUCCESS'] app['promotions'][sr] = (len(sr_history), len(successes)) logging.info(f"collecting merge activity for {app_name}") app['merge_activity'] = {} code_repos = [ c['url'] for c in app['codeComponents'] if c['resource'] == 'upstream' ] for cr in code_repos: cr_history = build_master_build_history.get(cr) if not cr_history: continue successes = [h for h in cr_history if h == 'SUCCESS'] app['merge_activity'][cr] = (len(cr_history), len(successes)) logging.info(f"collecting vulnerabilities information for {app_name}") app_namespaces = [] for namespace in namespaces: if namespace['app']['name'] != app['name']: continue app_namespaces.append(namespace) app_metrics = {} for family in text_string_to_metric_families(metrics): for sample in family.samples: if sample.name != 'imagemanifestvuln_total': continue for app_namespace in app_namespaces: cluster = sample.labels['cluster'] if app_namespace['cluster']['name'] != cluster: continue namespace = sample.labels['namespace'] if app_namespace['name'] != namespace: continue severity = sample.labels['severity'] if cluster not in app_metrics: app_metrics[cluster] = {} if namespace not in app_metrics[cluster]: app_metrics[cluster][namespace] = {} if severity not in app_metrics[cluster][namespace]: value = int(sample.value) app_metrics[cluster][namespace][severity] = value app['container_vulnerabilities'] = app_metrics return apps
class QuayMirror: GCR_PROJECT_CATALOG_QUERY = """ { projects: gcp_projects_v1 { name pushCredentials { path field } } } """ GCR_REPOS_QUERY = """ { apps: apps_v1 { gcrRepos { project { name } items { name mirror { url pullCredentials { path field } tags tagsExclude } } } } } """ def __init__(self, dry_run=False): self.dry_run = dry_run self.gqlapi = gql.get_api() settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=settings) self.skopeo_cli = Skopeo(dry_run) self.push_creds = self._get_push_creds() def run(self): sync_tasks = self.process_sync_tasks() for org, data in sync_tasks.items(): for item in data: try: self.skopeo_cli.copy(src_image=item['mirror_url'], src_creds=item['mirror_creds'], dst_image=item['image_url'], dest_creds=self.push_creds[org]) except SkopeoCmdError as details: _LOG.error('[%s]', details) def process_repos_query(self): result = self.gqlapi.query(self.GCR_REPOS_QUERY) summary = defaultdict(list) for app in result['apps']: gcr_repos = app.get('gcrRepos') if gcr_repos is None: continue for gcr_repo in gcr_repos: project = gcr_repo['project']['name'] server_url = gcr_repo['project'].get('serverUrl') or 'gcr.io' for item in gcr_repo['items']: if item['mirror'] is None: continue summary[project].append({ 'name': item["name"], 'mirror': item['mirror'], 'server_url': server_url }) return summary @staticmethod def sync_tag(tags, tags_exclude, candidate): if tags is not None: for tag in tags: if re.match(tag, candidate): return True # When tags is defined, we don't look at # tags_exclude return False if tags_exclude is not None: for tag_exclude in tags_exclude: if re.match(tag_exclude, candidate): return False return True # Both tags and tags_exclude are None, so # tag must be synced return True def process_sync_tasks(self): eight_hours = 28800 # 60 * 60 * 8 is_deep_sync = self._is_deep_sync(interval=eight_hours) summary = self.process_repos_query() sync_tasks = defaultdict(list) for org, data in summary.items(): for item in data: image = Image(f'{item["server_url"]}/{org}/{item["name"]}') mirror_url = item['mirror']['url'] username = None password = None mirror_creds = None if item['mirror']['pullCredentials'] is not None: pull_credentials = item['mirror']['pullCredentials'] raw_data = self.secret_reader.read_all(pull_credentials) username = raw_data["user"] password = raw_data["token"] mirror_creds = f'{username}:{password}' image_mirror = Image(mirror_url, username=username, password=password) tags = item['mirror'].get('tags') tags_exclude = item['mirror'].get('tagsExclude') for tag in image_mirror: if not self.sync_tag(tags=tags, tags_exclude=tags_exclude, candidate=tag): continue upstream = image_mirror[tag] downstream = image[tag] if tag not in image: _LOG.debug('Image %s and mirror %s are out off sync', downstream, upstream) sync_tasks[org].append({ 'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream) }) continue # Deep (slow) check only in non dry-run mode if self.dry_run: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue # Deep (slow) check only from time to time if not is_deep_sync: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue try: if downstream == upstream: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue except ImageComparisonError as details: _LOG.error('[%s]', details) continue _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) sync_tasks[org].append({ 'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream) }) return sync_tasks def _is_deep_sync(self, interval): control_file_name = 'qontract-reconcile-gcr-mirror.timestamp' control_file_path = os.path.join(tempfile.gettempdir(), control_file_name) try: with open(control_file_path, 'r') as file_obj: last_deep_sync = float(file_obj.read()) except FileNotFoundError: self._record_timestamp(control_file_path) return True next_deep_sync = last_deep_sync + interval if time.time() >= next_deep_sync: self._record_timestamp(control_file_path) return True return False @staticmethod def _record_timestamp(path): with open(path, 'w') as file_object: file_object.write(str(time.time())) def _get_push_creds(self): result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY) creds = {} for project_data in result['projects']: push_secret = project_data['pushCredentials'] if push_secret is None: continue raw_data = self.secret_reader.read_all(push_secret) project = project_data['name'] token = base64.b64decode(raw_data["token"]).decode() creds[project] = f'{raw_data["user"]}:{token}' return creds
class EcrMirror: def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance['identifier'] account = instance['account'] region = instance.get('region') self.aws_cli = AWSApi(thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f'{self.ecr_username}:{self.ecr_password}' self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance['mirror']['pullCredentials'] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f'{self.image_username}:{self.image_password}' def run(self): if self.error: return ecr_mirror = Image(self.ecr_uri, username=self.ecr_username, password=self.ecr_password) image = Image(self.instance['mirror']['url'], username=self.image_username, password=self.image_password) LOG.debug('[checking %s -> %s]', image, ecr_mirror) for tag in image: if tag not in ecr_mirror: try: self.skopeo_cli.copy(src_image=image[tag], src_creds=self.image_auth, dst_image=ecr_mirror[tag], dest_creds=self.ecr_auth) except SkopeoCmdError as details: LOG.error('[%s]', details) def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]['resourcesDefaultRegion'] auth_token = f'{account}/{region}' data = self.aws_cli.auth_tokens[auth_token] auth_data = data['authorizationData'][0] token = auth_data['authorizationToken'] password = base64.b64decode(token).decode('utf-8').split(':')[1] return 'AWS', password def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]['ecr']: if repo['repositoryName'] == repository: return repo['repositoryUri'] @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if 'name' not in account_info: continue if account_info['name'] != account: continue return account_info
class AWSApi: """Wrapper around AWS SDK""" def __init__(self, thread_pool_size, accounts, settings=None, init_ecr_auth_tokens=False, init_users=True): self.thread_pool_size = thread_pool_size self.secret_reader = SecretReader(settings=settings) self.init_sessions_and_resources(accounts) if init_ecr_auth_tokens: self.init_ecr_auth_tokens(accounts) if init_users: self.init_users() self._lock = Lock() self.resource_types = \ ['s3', 'sqs', 'dynamodb', 'rds', 'rds_snapshots'] # store the app-interface accounts in a dictionary indexed by name self.accounts = {acc['name']: acc for acc in accounts} # Setup caches on the instance itself to avoid leak # https://stackoverflow.com/questions/33672412/python-functools-lru-cache-with-class-methods-release-object # using @lru_cache decorators on methods would lek AWSApi instances # since the cache keeps a reference to self. self._account_ec2_client = functools.lru_cache()( self._account_ec2_client) self._get_assumed_role_client = functools.lru_cache()( self._get_assumed_role_client) self.get_account_vpcs = functools.lru_cache()(self.get_account_vpcs) self.get_vpc_route_tables = functools.lru_cache()( self.get_vpc_route_tables) self.get_vpc_subnets = functools.lru_cache()(self.get_vpc_subnets) self.get_vpc_default_sg_id = functools.lru_cache()( self.get_vpc_default_sg_id) self.get_transit_gateways = functools.lru_cache()( self.get_transit_gateways) self.get_transit_gateway_vpc_attachments = functools.lru_cache()( self.get_transit_gateway_vpc_attachments) def init_sessions_and_resources(self, accounts: Iterable[Account]): results = threaded.run(self.get_tf_secrets, accounts, self.thread_pool_size) self.sessions: Dict[str, Session] = {} self.resources: Dict[str, Any] = {} for account, secret in results: access_key = secret['aws_access_key_id'] secret_key = secret['aws_secret_access_key'] region_name = secret['region'] session = Session( aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, ) self.sessions[account] = session self.resources[account] = {} def get_session(self, account: str) -> Session: return self.sessions[account] # pylint: disable=method-hidden def _account_ec2_client(self, account_name: str, region_name: Optional[str] = None) -> EC2Client: session = self.get_session(account_name) region = region_name if region_name else session.region_name return session.client('ec2', region_name=region) def get_tf_secrets(self, account): account_name = account['name'] automation_token = account['automationToken'] secret = self.secret_reader.read_all(automation_token) return (account_name, secret) def init_users(self): self.users = {} for account, s in self.sessions.items(): iam = s.client('iam') users = self.paginate(iam, 'list_users', 'Users') users = [u['UserName'] for u in users] self.users[account] = users def simulate_deleted_users(self, io_dir): src_integrations = ['terraform_resources', 'terraform_users'] if not os.path.exists(io_dir): return for i in src_integrations: file_path = os.path.join(io_dir, i + '.json') if not os.path.exists(file_path): continue with open(file_path, 'r') as f: deleted_users = json.load(f) for deleted_user in deleted_users: delete_from_account = deleted_user['account'] delete_user = deleted_user['user'] self.users[delete_from_account].remove(delete_user) def map_resources(self): threaded.run(self.map_resource, self.resource_types, self.thread_pool_size) def map_resource(self, resource_type): if resource_type == 's3': self.map_s3_resources() elif resource_type == 'sqs': self.map_sqs_resources() elif resource_type == 'dynamodb': self.map_dynamodb_resources() elif resource_type == 'rds': self.map_rds_resources() elif resource_type == 'rds_snapshots': self.map_rds_snapshots() elif resource_type == 'route53': self.map_route53_resources() else: raise InvalidResourceTypeError(resource_type) def map_s3_resources(self): for account, s in self.sessions.items(): s3 = s.client('s3') buckets_list = s3.list_buckets() if 'Buckets' not in buckets_list: continue buckets = [b['Name'] for b in buckets_list['Buckets']] self.set_resouces(account, 's3', buckets) buckets_without_owner = \ self.get_resources_without_owner(account, buckets) unfiltered_buckets = \ self.custom_s3_filter(account, s3, buckets_without_owner) self.set_resouces(account, 's3_no_owner', unfiltered_buckets) def map_sqs_resources(self): for account, s in self.sessions.items(): sqs = s.client('sqs') queues_list = sqs.list_queues() if 'QueueUrls' not in queues_list: continue queues = queues_list['QueueUrls'] self.set_resouces(account, 'sqs', queues) queues_without_owner = \ self.get_resources_without_owner(account, queues) unfiltered_queues = \ self.custom_sqs_filter(account, sqs, queues_without_owner) self.set_resouces(account, 'sqs_no_owner', unfiltered_queues) def map_dynamodb_resources(self): for account, s in self.sessions.items(): dynamodb = s.client('dynamodb') tables = self.paginate(dynamodb, 'list_tables', 'TableNames') self.set_resouces(account, 'dynamodb', tables) tables_without_owner = \ self.get_resources_without_owner(account, tables) unfiltered_tables = \ self.custom_dynamodb_filter( account, s, dynamodb, tables_without_owner ) self.set_resouces(account, 'dynamodb_no_owner', unfiltered_tables) def map_rds_resources(self): for account, s in self.sessions.items(): rds = s.client('rds') results = \ self.paginate(rds, 'describe_db_instances', 'DBInstances') instances = [t['DBInstanceIdentifier'] for t in results] self.set_resouces(account, 'rds', instances) instances_without_owner = \ self.get_resources_without_owner(account, instances) unfiltered_instances = \ self.custom_rds_filter(account, rds, instances_without_owner) self.set_resouces(account, 'rds_no_owner', unfiltered_instances) def map_rds_snapshots(self): self.wait_for_resource('rds') for account, s in self.sessions.items(): rds = s.client('rds') results = \ self.paginate(rds, 'describe_db_snapshots', 'DBSnapshots') snapshots = [t['DBSnapshotIdentifier'] for t in results] self.set_resouces(account, 'rds_snapshots', snapshots) snapshots_without_db = [ t['DBSnapshotIdentifier'] for t in results if t['DBInstanceIdentifier'] not in self.resources[account]['rds'] ] unfiltered_snapshots = \ self.custom_rds_snapshot_filter(account, rds, snapshots_without_db) self.set_resouces(account, 'rds_snapshots_no_owner', unfiltered_snapshots) def map_route53_resources(self): for account, s in self.sessions.items(): client = s.client('route53') results = \ self.paginate(client, 'list_hosted_zones', 'HostedZones') zones = list(results) for zone in zones: results = \ self.paginate(client, 'list_resource_record_sets', 'ResourceRecordSets', {'HostedZoneId': zone['Id']}) zone['records'] = results self.set_resouces(account, 'route53', zones) def map_ecr_resources(self): for account, s in self.sessions.items(): client = s.client('ecr') repositories = self.paginate(client=client, method='describe_repositories', key='repositories') self.set_resouces(account, 'ecr', repositories) @staticmethod def paginate(client, method, key, params={}): """ paginate returns an aggregated list of the specified key from all pages returned by executing the client's specified method.""" paginator = client.get_paginator(method) return [ values for page in paginator.paginate(**params) for values in page.get(key, []) ] def wait_for_resource(self, resource): """ wait_for_resource waits until the specified resource type is ready for all accounts. When we have more resource types then threads, this function will need to change to a dependency graph.""" wait = True while wait: wait = False for account in self.sessions: if self.resources[account].get(resource) is None: wait = True if wait: time.sleep(2) def set_resouces(self, account, key, value): with self._lock: self.resources[account][key] = value def get_resources_without_owner(self, account, resources): return [r for r in resources if not self.has_owner(account, r)] def has_owner(self, account, resource): has_owner = False for u in self.users[account]: if resource.lower().startswith(u.lower()): has_owner = True break if '://' in resource: if resource.split('/')[-1].startswith(u.lower()): has_owner = True break return has_owner def custom_s3_filter(self, account, s3, buckets): type = 's3 bucket' unfiltered_buckets = [] for b in buckets: try: tags = s3.get_bucket_tagging(Bucket=b) except botocore.exceptions.ClientError: tags = {} if not self.should_filter(account, type, b, tags, 'TagSet'): unfiltered_buckets.append(b) return unfiltered_buckets def custom_sqs_filter(self, account, sqs, queues): type = 'sqs queue' unfiltered_queues = [] for q in queues: tags = sqs.list_queue_tags(QueueUrl=q) if not self.should_filter(account, type, q, tags, 'Tags'): unfiltered_queues.append(q) return unfiltered_queues def custom_dynamodb_filter(self, account, session, dynamodb, tables): type = 'dynamodb table' dynamodb_resource = session.resource('dynamodb') unfiltered_tables = [] for t in tables: table_arn = dynamodb_resource.Table(t).table_arn tags = dynamodb.list_tags_of_resource(ResourceArn=table_arn) if not self.should_filter(account, type, t, tags, 'Tags'): unfiltered_tables.append(t) return unfiltered_tables def custom_rds_filter(self, account, rds, instances): type = 'rds instance' unfiltered_instances = [] for i in instances: instance = rds.describe_db_instances(DBInstanceIdentifier=i) instance_arn = instance['DBInstances'][0]['DBInstanceArn'] tags = rds.list_tags_for_resource(ResourceName=instance_arn) if not self.should_filter(account, type, i, tags, 'TagList'): unfiltered_instances.append(i) return unfiltered_instances def custom_rds_snapshot_filter(self, account, rds, snapshots): type = 'rds snapshots' unfiltered_snapshots = [] for s in snapshots: snapshot = rds.describe_db_snapshots(DBSnapshotIdentifier=s) snapshot_arn = snapshot['DBSnapshots'][0]['DBSnapshotArn'] tags = rds.list_tags_for_resource(ResourceName=snapshot_arn) if not self.should_filter(account, type, s, tags, 'TagList'): unfiltered_snapshots.append(s) return unfiltered_snapshots def should_filter(self, account, resource_type, resource_name, resource_tags, tags_key): if self.resource_has_special_name(account, resource_type, resource_name): return True if tags_key in resource_tags: tags = resource_tags[tags_key] if self.resource_has_special_tags(account, resource_type, resource_name, tags): return True return False @staticmethod def resource_has_special_name(account, type, resource): skip_msg = '[{}] skipping {} '.format(account, type) + \ '({} related) {}' ignore_names = { 'production': ['prod'], 'stage': ['stage', 'staging'], 'terraform': ['terraform', '-tf-'], } for msg, tags in ignore_names.items(): for tag in tags: if tag.lower() in resource.lower(): logging.debug(skip_msg.format(msg, resource)) return True return False def resource_has_special_tags(self, account, type, resource, tags): skip_msg = '[{}] skipping {} '.format(account, type) + \ '({}={}) {}' ignore_tags = { 'ENV': ['prod', 'stage', 'staging'], 'environment': ['prod', 'stage', 'staging'], 'owner': ['app-sre'], 'managed_by_integration': ['terraform_resources', 'terraform_users'], 'aws_gc_hands_off': ['true'], } for tag, ignore_values in ignore_tags.items(): for ignore_value in ignore_values: value = self.get_tag_value(tags, tag) if ignore_value.lower() in value.lower(): logging.debug(skip_msg.format(tag, value, resource)) return True return False @staticmethod def get_tag_value(tags, tag): if isinstance(tags, dict): return tags.get(tag, '') elif isinstance(tags, list): for t in tags: if t['Key'] == tag: return t['Value'] return '' def delete_resources_without_owner(self, dry_run): for account, s in self.sessions.items(): for rt in self.resource_types: for r in self.resources[account].get(rt + '_no_owner', []): logging.info(['delete_resource', account, rt, r]) if not dry_run: self.delete_resource(s, rt, r) def delete_resource(self, session, resource_type, resource_name): if resource_type == 's3': resource = session.resource(resource_type) self.delete_bucket(resource, resource_name) elif resource_type == 'sqs': client = session.client(resource_type) self.delete_queue(client, resource_name) elif resource_type == 'dynamodb': resource = session.resource(resource_type) self.delete_table(resource, resource_name) elif resource_type == 'rds': client = session.client(resource_type) self.delete_instance(client, resource_name) elif resource_type == 'rds_snapshots': client = session.client(resource_type) self.delete_snapshot(client, resource_name) else: raise InvalidResourceTypeError(resource_type) @staticmethod def delete_bucket(s3, bucket_name): bucket = s3.Bucket(bucket_name) bucket.object_versions.delete() bucket.delete() @staticmethod def delete_queue(sqs, queue_url): sqs.delete_queue(QueueUrl=queue_url) @staticmethod def delete_table(dynamodb, table_name): table = dynamodb.Table(table_name) table.delete() @staticmethod def delete_instance(rds, instance_name): rds.delete_db_instance(DBInstanceIdentifier=instance_name, SkipFinalSnapshot=True, DeleteAutomatedBackups=True) @staticmethod def delete_snapshot(rds, snapshot_identifier): rds.delete_db_snapshot(DBSnapshotIdentifier=snapshot_identifier) @staticmethod def determine_key_type(iam, user): tags = iam.list_user_tags(UserName=user)['Tags'] managed_by_integration_tag = \ [t['Value'] for t in tags if t['Key'] == 'managed_by_integration'] # if this key belongs to a user without tags, i.e. not # managed by an integration, this key is probably created # manually. disable it to leave a trace if not managed_by_integration_tag: return 'unmanaged' # if this key belongs to a user created by the # 'terraform-users' integration, we just delete the key if managed_by_integration_tag[0] == 'terraform_users': return 'user' # if this key belongs to a user created by the # 'terraform-resources' integration, we remove # the key from terraform state and let it create # a new one on its own if managed_by_integration_tag[0] == 'terraform_resources': return 'service_account' huh = 'unrecognized managed_by_integration tag: {}'.format( managed_by_integration_tag[0]) raise InvalidResourceTypeError(huh) def delete_keys(self, dry_run, keys_to_delete, working_dirs, disable_service_account_keys): error = False users_keys = self.get_users_keys() for account, s in self.sessions.items(): iam = s.client('iam') keys = keys_to_delete.get(account, []) for key in keys: user_and_user_keys = [ (user, user_keys) for user, user_keys in users_keys[account].items() if key in user_keys ] if not user_and_user_keys: continue # unpack single item from sequence # since only a single user can have a given key [user_and_user_keys] = user_and_user_keys user = user_and_user_keys[0] user_keys = user_and_user_keys[1] key_type = self.determine_key_type(iam, user) key_status = self.get_user_key_status(iam, user, key) if key_type == 'unmanaged' and key_status == 'Active': logging.info(['disable_key', account, user, key]) if not dry_run: iam.update_access_key(UserName=user, AccessKeyId=key, Status='Inactive') elif key_type == 'user': logging.info(['delete_key', account, user, key]) if not dry_run: iam.delete_access_key(UserName=user, AccessKeyId=key) elif key_type == 'service_account': # if key is disabled - delete it # this will happen after terraform-resources ran, # provisioned a new key, updated the output Secret, # recycled the pods and disabled the key. if key_status == 'Inactive': logging.info( ['delete_inactive_key', account, user, key]) if not dry_run: iam.delete_access_key(UserName=user, AccessKeyId=key) continue # if key is active and it is the only one - # remove it from terraform state. terraform-resources # will provision a new one. # may be a race condition here. TODO: check it if len(user_keys) == 1: logging.info(['remove_from_state', account, user, key]) if not dry_run: terraform.state_rm_access_key( working_dirs, account, user) # if user has 2 keys and we remove the key from # terraform state, terraform-resources will not # be able to provision a new key - limbo. # this state should happen when terraform-resources # is running, provisioned a new key, # but did not disable the old key yet. if len(user_keys) == 2: # if true, this is a call made by terraform-resources # itself. disable the key and proceed. the key will be # deleted in a following iteration of aws-iam-keys. if disable_service_account_keys: logging.info(['disable_key', account, user, key]) if not dry_run: iam.update_access_key(UserName=user, AccessKeyId=key, Status='Inactive') else: msg = \ 'user {} has 2 keys, skipping to avoid error' logging.error(msg.format(user)) error = True return error def get_users_keys(self): users_keys = {} for account, s in self.sessions.items(): iam = s.client('iam') users_keys[account] = { user: self.get_user_keys(iam, user) for user in self.users[account] } return users_keys def reset_password(self, account, user_name): s = self.sessions[account] iam = s.client('iam') iam.delete_login_profile(UserName=user_name) @staticmethod def get_user_keys(iam, user): key_list = iam.list_access_keys(UserName=user)['AccessKeyMetadata'] return [uk['AccessKeyId'] for uk in key_list] @staticmethod def get_user_key_status(iam, user, key): key_list = iam.list_access_keys(UserName=user)['AccessKeyMetadata'] return [k['Status'] for k in key_list if k['AccessKeyId'] == key][0] def get_support_cases(self): all_support_cases = {} for account, s in self.sessions.items(): if not self.accounts[account].get('premiumSupport'): continue try: support = s.client('support') support_cases = support.describe_cases( includeResolvedCases=True, includeCommunications=True)['cases'] all_support_cases[account] = support_cases except Exception as e: msg = '[{}] error getting support cases. details: {}' logging.error(msg.format(account, str(e))) return all_support_cases def init_ecr_auth_tokens(self, accounts): accounts_with_ecr = [a for a in accounts if a.get('ecrs')] if not accounts_with_ecr: return auth_tokens = {} results = threaded.run(self.get_tf_secrets, accounts_with_ecr, self.thread_pool_size) account_secrets = dict(results) for account in accounts_with_ecr: account_name = account['name'] account_secret = account_secrets[account_name] access_key = account_secret['aws_access_key_id'] secret_key = account_secret['aws_secret_access_key'] ecrs = account['ecrs'] for ecr in ecrs: region_name = ecr['region'] session = Session( aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, ) client = session.client('ecr') token = client.get_authorization_token() auth_tokens[f"{account_name}/{region_name}"] = token self.auth_tokens = auth_tokens @staticmethod def _get_account_assume_data(account: Account) -> Tuple[str, str, str]: """ returns mandatory data to be able to assume a role with this account: (account_name, assume_role, assume_region) """ required_keys = \ ['name', 'assume_role', 'assume_region'] ok = all(elem in account.keys() for elem in required_keys) if not ok: account_name = account.get('name') raise KeyError( '[{}] account is missing required keys'.format(account_name)) return (account['name'], account['assume_role'], account['assume_region']) def _get_assume_role_session(self, account_name: str, assume_role: str, assume_region: str) -> Session: """ Returns a session for a supplied role to assume: :param name: name of the AWS account :param assume_role: role to assume to get access to the cluster's AWS account :param assume_region: region in which to operate """ session = self.get_session(account_name) sts = session.client('sts') if not assume_role: raise MissingARNError( f'Could not find Role ARN {assume_role} on account ' f'{account_name}. This is likely caused by a missing ' 'awsInfrastructureAccess section.') role_name = assume_role.split('/')[1] response = sts.assume_role(RoleArn=assume_role, RoleSessionName=role_name) credentials = response['Credentials'] assumed_session = Session( aws_access_key_id=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_session_token=credentials['SessionToken'], region_name=assume_region) return assumed_session # pylint: disable=method-hidden def _get_assumed_role_client(self, account_name: str, assume_role: str, assume_region: str) -> EC2Client: assumed_session = self._get_assume_role_session( account_name, assume_role, assume_region) return assumed_session.client('ec2') @staticmethod # pylint: disable=method-hidden def get_account_vpcs(ec2: EC2Client) -> List[VpcTypeDef]: vpcs = ec2.describe_vpcs() return vpcs.get('Vpcs', []) # filters a list of aws resources according to tags @staticmethod def filter_on_tags(items: Iterable[Any], tags: Mapping[str, str] = {}) \ -> List[Any]: res = [] for item in items: tags_dict = {t['Key']: t['Value'] for t in item.get('Tags', [])} if all(tags_dict.get(k) == values for k, values in tags.items()): res.append(item) return res @staticmethod # pylint: disable=method-hidden def get_vpc_route_tables(vpc_id: str, ec2: EC2Client) \ -> List[RouteTableTypeDef]: rts = ec2.describe_route_tables(Filters=[{ 'Name': 'vpc-id', 'Values': [vpc_id] }]) return rts.get('RouteTables', []) @staticmethod # pylint: disable=method-hidden def get_vpc_subnets(vpc_id: str, ec2: EC2Client) \ -> List[SubnetTypeDef]: subnets = ec2.describe_subnets(Filters=[{ 'Name': 'vpc-id', 'Values': [vpc_id] }]) return subnets.get('Subnets', []) def get_cluster_vpc_details(self, account, route_tables=False, subnets=False): """ Returns a cluster VPC details: - VPC ID - Route table IDs (optional) - Subnets list including Subnet ID and Subnet Availability zone :param account: a dictionary containing the following keys: - name - name of the AWS account - assume_role - role to assume to get access to the cluster's AWS account - assume_region - region in which to operate - assume_cidr - CIDR block of the cluster to use to find the matching VPC """ assume_role_data = self._get_account_assume_data(account) assumed_ec2 = self._get_assumed_role_client(*assume_role_data) vpcs = self.get_account_vpcs(assumed_ec2) vpc_id = None for vpc in vpcs: if vpc['CidrBlock'] == account['assume_cidr']: vpc_id = vpc['VpcId'] break route_table_ids = None subnets_id_az = None if vpc_id: if route_tables: vpc_route_tables = \ self.get_vpc_route_tables(vpc_id, assumed_ec2) route_table_ids = [ rt['RouteTableId'] for rt in vpc_route_tables ] if subnets: vpc_subnets = self.get_vpc_subnets(vpc_id, assumed_ec2) subnets_id_az = \ [ { 'id': s['SubnetId'], 'az': s['AvailabilityZone'] } for s in vpc_subnets ] return vpc_id, route_table_ids, subnets_id_az def get_cluster_nat_gateways_egress_ips(self, account): assumed_role_data = self._get_account_assume_data(account) assumed_ec2 = self._get_assumed_role_client(*assumed_role_data) nat_gateways = assumed_ec2.describe_nat_gateways() egress_ips = set() for nat in nat_gateways.get('NatGateways'): for address in nat['NatGatewayAddresses']: egress_ips.add(address['PublicIp']) return egress_ips def get_vpcs_details(self, account, tags=None, route_tables=False): results = [] ec2 = self._account_ec2_client(account['name']) regions = [r['RegionName'] for r in ec2.describe_regions()['Regions']] for region_name in regions: ec2 = self._account_ec2_client(account['name'], region_name) vpcs = self.get_account_vpcs(ec2) vpcs = self.filter_on_tags(vpcs, tags) for vpc in vpcs: vpc_id = vpc['VpcId'] cidr_block = vpc['CidrBlock'] route_table_ids = None if route_tables: vpc_route_tables = self.get_vpc_route_tables(vpc_id, ec2) route_table_ids = [ rt['RouteTableId'] for rt in vpc_route_tables ] item = { 'vpc_id': vpc_id, 'region': region_name, 'cidr_block': cidr_block, 'route_table_ids': route_table_ids, } results.append(item) return results @staticmethod # pylint: disable=method-hidden def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> Optional[str]: vpc_security_groups = ec2.describe_security_groups( Filters=[{ 'Name': 'vpc-id', 'Values': [vpc_id] }, { 'Name': 'group-name', 'Values': ['default'] }]) # there is only one default for sg in vpc_security_groups.get('SecurityGroups', []): return sg['GroupId'] return None @staticmethod # pylint: disable=method-hidden def get_transit_gateways(ec2: EC2Client) -> List[TransitGatewayTypeDef]: tgws = ec2.describe_transit_gateways() return tgws.get('TransitGateways', []) def get_tgw_default_route_table_id(self, ec2: EC2Client, tgw_id: str, tags: Mapping[str, str]) \ -> Optional[str]: tgws = self.get_transit_gateways(ec2) tgws = self.filter_on_tags(tgws, tags) # we know the party TGW exists, so we can be # a little less catious about getting it [tgw] = [t for t in tgws if t['TransitGatewayId'] == tgw_id] tgw_options = tgw['Options'] tgw_has_route_table = \ tgw_options['DefaultRouteTableAssociation'] == 'enable' # currently only adding routes # to the default route table if tgw_has_route_table: return tgw_options['AssociationDefaultRouteTableId'] return None @staticmethod # pylint: disable=method-hidden def get_transit_gateway_vpc_attachments(tgw_id: str, ec2: EC2Client) \ -> List[TransitGatewayVpcAttachmentTypeDef]: atts = ec2.describe_transit_gateway_vpc_attachments( Filters=[{ 'Name': 'transit-gateway-id', 'Values': [tgw_id] }]) return atts.get('TransitGatewayVpcAttachments', []) def get_tgws_details(self, account, region_name, routes_cidr_block, tags=None, route_tables=False, security_groups=False): results = [] ec2 = self._account_ec2_client(account['name'], region_name) tgws = ec2.describe_transit_gateways(Filters=[{ 'Name': f'tag:{k}', 'Values': [v] } for k, v in tags.items()]) for tgw in tgws.get('TransitGateways'): tgw_id = tgw['TransitGatewayId'] tgw_arn = tgw['TransitGatewayArn'] item = { 'tgw_id': tgw_id, 'tgw_arn': tgw_arn, 'region': region_name, } if route_tables or security_groups: # both routes and rules are provisioned for resources # that are indirectly attached to the TGW. # routes are provisioned for route tables that belong # to TGWs which are peered to the TGW we are currently # handling. routes = [] # rules are provisioned for security groups that belong # to VPCs which are attached to the TGW we are currently # handling AND to TGWs which are peered to it. rules = [] # this will require to iterate over all reachable TGWs attachments = \ ec2.describe_transit_gateway_peering_attachments( Filters=[ {'Name': 'transit-gateway-id', 'Values': [tgw_id]} ] ) for a in attachments.get('TransitGatewayPeeringAttachments'): tgw_attachment_id = a['TransitGatewayAttachmentId'] tgw_attachment_state = a['State'] if tgw_attachment_state != 'available': continue # we don't care who is who, so let's iterate over parties attachment_parties = \ [a['RequesterTgwInfo'], a['AccepterTgwInfo']] for party in attachment_parties: party_tgw_id = party['TransitGatewayId'] party_region = party['Region'] party_ec2 = self._account_ec2_client( account['name'], party_region) # the TGW route table is automatically populated # with the peered VPC cidr block. # however, to achieve global routing across peered # TGWs in different regions, we need to find all # peering attachments in different regions and collect # the data to later create a route in each peered TGW # in a different region. this will require getting: # - cluster cidr block # - transit gateway attachment id # - transit gateway route table id # we will also pass some additional information: # - transit gateway id # - transit gateway region if route_tables: # don't act on yourself and # routes are propogated within the same region if party_tgw_id != tgw_id and \ party_region != region_name: party_tgw_route_table_id = \ self.get_tgw_default_route_table_id( party_ec2, party_tgw_id, tags) if party_tgw_route_table_id is not None: # that's it, we have all # the information we need route_item = { 'cidr_block': routes_cidr_block, 'tgw_attachment_id': tgw_attachment_id, 'tgw_id': party_tgw_id, 'tgw_route_table_id': party_tgw_route_table_id, 'region': party_region } routes.append(route_item) # once all the routing is in place, we need to allow # connections in security groups. # in TGW, we need to allow the rules in the VPCs # associated to the TGWs that need to accept the # traffic. we need to collect data about the vpc # attachments for the TGWs, and for each VPC get # the details of it's default securiry group. # this will require getting: # - cluster cidr block # - security group id # we will also pass some additional information: # - vpc id # - vpc region if security_groups: vpc_attachments = \ self.get_transit_gateway_vpc_attachments( party_tgw_id, party_ec2) for va in vpc_attachments: vpc_attachment_vpc_id = va['VpcId'] vpc_attachment_state = va['State'] if vpc_attachment_state != 'available': continue sg_id = self.get_vpc_default_sg_id( vpc_attachment_vpc_id, party_ec2) if sg_id is not None: # that's it, we have all # the information we need rule_item = { 'cidr_block': routes_cidr_block, 'security_group_id': sg_id, 'vpc_id': vpc_attachment_vpc_id, 'region': party_region } rules.append(rule_item) if route_tables: item['routes'] = routes if security_groups: item['rules'] = rules results.append(item) return results def get_route53_zones(self): """ Return a list of (str, dict) representing Route53 DNS zones per account :return: route53 dns zones per account :rtype: list of (str, dict) """ return { account: self.resources.get(account, {}).get('route53', []) for account, _ in self.sessions.items() } def create_route53_zone(self, account_name, zone_name): """ Create a Route53 DNS zone :param account_name: the account name to operate on :param zone_name: name of the zone to create :type account_name: str :type zone_name: str """ session = self.get_session(account_name) client = session.client('route53') try: caller_ref = f"{datetime.now()}" client.create_hosted_zone( Name=zone_name, CallerReference=caller_ref, HostedZoneConfig={ 'Comment': 'Managed by App-Interface', }, ) except client.exceptions.InvalidDomainName: logging.error(f'[{account_name}] invalid domain name {zone_name}') except client.exceptions.HostedZoneAlreadyExists: logging.error( f'[{account_name}] hosted zone already exists: {zone_name}') except client.exceptions.TooManyHostedZones: logging.error(f'[{account_name}] too many hosted zones in account') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}') def delete_route53_zone(self, account_name, zone_id): """ Delete a Route53 DNS zone :param account_name: the account name to operate on :param zone_id: aws zone id of the zone to delete :type account_name: str :type zone_id: str """ session = self.get_session(account_name) client = session.client('route53') try: client.delete_hosted_zone(Id=zone_id) except client.exceptions.NoSuchHostedZone: logging.error(f'[{account_name}] Error trying to delete ' f'unknown DNS zone {zone_id}') except client.exceptions.HostedZoneNotEmpty: logging.error(f'[{account_name}] Cannot delete DNS zone that ' f'is not empty {zone_id}') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}') def delete_route53_record(self, account_name, zone_id, awsdata): """ Delete a Route53 DNS zone record :param account_name: the account name to operate on :param zone_id: aws zone id of the zone to operate on :param awsdata: aws record data of the record to delete :type account_name: str :type zone_id: str :type awsdata: dict """ session = self.get_session(account_name) client = session.client('route53') try: client.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch={ 'Changes': [{ 'Action': 'DELETE', 'ResourceRecordSet': awsdata, }] }) except client.exceptions.NoSuchHostedZone: logging.error(f'[{account_name}] Error trying to delete record: ' f'unknown DNS zone {zone_id}') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}') def upsert_route53_record(self, account_name, zone_id, recordset): """ Upsert a Route53 DNS zone record :param account_name: the account name to operate on :param zone_id: aws zone id of the zone to operate on :param recordset: aws record data of the record to create or update :type account_name: str :type zone_id: str :type recordset: dict """ session = self.get_session(account_name) client = session.client('route53') try: client.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch={ 'Changes': [{ 'Action': 'UPSERT', 'ResourceRecordSet': recordset, }] }) except client.exceptions.NoSuchHostedZone: logging.error(f'[{account_name}] Error trying to delete record: ' f'unknown DNS zone {zone_id}') except Exception as e: logging.error(f'[{account_name}] unhandled exception: {e}')
def get_apps_data(date, month_delta=1): apps = queries.get_apps() jjb, _ = init_jjb() saas_jobs = jjb.get_all_jobs(job_types=['saas-deploy', 'promote-to-prod']) build_master_jobs = jjb.get_all_jobs(job_types=['build-master']) jenkins_map = jenkins_base.get_jenkins_map() time_limit = date - relativedelta(months=month_delta) timestamp_limit = \ int(time_limit.replace(tzinfo=timezone.utc).timestamp()) saas_build_history = \ get_build_history(jenkins_map, saas_jobs, timestamp_limit) build_master_build_history = \ get_build_history(jenkins_map, build_master_jobs, timestamp_limit) settings = queries.get_app_interface_settings() secret_reader = SecretReader(settings=settings) secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET}) dashdotdb_url = secret_content['url'] dashdotdb_user = secret_content['username'] dashdotdb_pass = secret_content['password'] metrics = requests.get(f'{dashdotdb_url}/api/v1/metrics', auth=(dashdotdb_user, dashdotdb_pass)).text namespaces = queries.get_namespaces() for app in apps: if not app['codeComponents']: continue app_name = app['name'] logging.info(f"collecting promotions for {app_name}") app['promotions'] = {} saas_repos = [ c['url'] for c in app['codeComponents'] if c['resource'] == 'saasrepo' ] for sr in saas_repos: sr_history = saas_build_history.get(sr) if not sr_history: continue successes = [h for h in sr_history if h == 'SUCCESS'] app['promotions'][sr] = (len(sr_history), len(successes)) logging.info(f"collecting merge activity for {app_name}") app['merge_activity'] = {} code_repos = [ c['url'] for c in app['codeComponents'] if c['resource'] == 'upstream' ] for cr in code_repos: cr_history = build_master_build_history.get(cr) if not cr_history: continue successes = [h for h in cr_history if h == 'SUCCESS'] app['merge_activity'][cr] = (len(cr_history), len(successes)) logging.info(f"collecting dashdotdb information for {app_name}") app_namespaces = [] for namespace in namespaces: if namespace['app']['name'] != app['name']: continue app_namespaces.append(namespace) vuln_mx = {} validt_mx = {} for family in text_string_to_metric_families(metrics): for sample in family.samples: if sample.name == 'imagemanifestvuln_total': for app_namespace in app_namespaces: cluster = sample.labels['cluster'] if app_namespace['cluster']['name'] != cluster: continue namespace = sample.labels['namespace'] if app_namespace['name'] != namespace: continue severity = sample.labels['severity'] if cluster not in vuln_mx: vuln_mx[cluster] = {} if namespace not in vuln_mx[cluster]: vuln_mx[cluster][namespace] = {} if severity not in vuln_mx[cluster][namespace]: value = int(sample.value) vuln_mx[cluster][namespace][severity] = value if sample.name == 'deploymentvalidation_total': for app_namespace in app_namespaces: cluster = sample.labels['cluster'] if app_namespace['cluster']['name'] != cluster: continue namespace = sample.labels['namespace'] if app_namespace['name'] != namespace: continue validation = sample.labels['validation'] # dvo: fail == 1, pass == 0, py: true == 1, false == 0 # so: ({false|pass}, {true|fail}) status = ('Passed', 'Failed')[sample.labels['status']] if cluster not in validt_mx: validt_mx[cluster] = {} if namespace not in validt_mx[cluster]: validt_mx[cluster][namespace] = {} if validation not in validt_mx[cluster][namespace]: validt_mx[cluster][namespace][validation] = {} if status not in validt_mx[cluster][namespace][ validation]: # noqa: E501 validt_mx[cluster][namespace][validation][ status] = {} # noqa: E501 value = int(sample.value) validt_mx[cluster][namespace][validation][ status] = value # noqa: E501 app['container_vulnerabilities'] = vuln_mx app['deployment_validations'] = validt_mx return apps
class QuayMirror: QUAY_ORG_CATALOG_QUERY = """ { quay_orgs: quay_orgs_v1 { name pushCredentials { path field } instance { name url } } } """ response_cache = InstrumentedCache(integration_name=QONTRACT_INTEGRATION, shards=sharding.SHARDS, shard_id=sharding.SHARD_ID) def __init__(self, dry_run=False): self.dry_run = dry_run self.gqlapi = gql.get_api() settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=settings) self.skopeo_cli = Skopeo(dry_run) self.push_creds = self._get_push_creds() def run(self): sync_tasks = self.process_sync_tasks() for org, data in sync_tasks.items(): for item in data: try: self.skopeo_cli.copy(src_image=item['mirror_url'], src_creds=item['mirror_creds'], dst_image=item['image_url'], dest_creds=self.push_creds[org]) except SkopeoCmdError as details: _LOG.error('[%s]', details) @classmethod def process_repos_query(cls): apps = queries.get_quay_repos() summary = defaultdict(list) for app in apps: quay_repos = app.get('quayRepos') if quay_repos is None: continue for quay_repo in quay_repos: org = quay_repo['org']['name'] instance = quay_repo['org']['instance']['name'] server_url = quay_repo['org']['instance']['url'] for item in quay_repo['items']: if item['mirror'] is None: continue mirror_image = Image(item['mirror']['url'], response_cache=cls.response_cache) if (mirror_image.registry == 'docker.io' and mirror_image.repository == 'library' and item['public']): _LOG.error( "Image %s can't be mirrored to a public " "quay repository.", mirror_image) sys.exit(ExitCodes.ERROR) org_key = OrgKey(instance, org) summary[org_key].append({ 'name': item["name"], 'mirror': item['mirror'], 'server_url': server_url }) return summary @staticmethod def sync_tag(tags, tags_exclude, candidate): if tags is not None: for tag in tags: if re.match(tag, candidate): return True # When tags is defined, we don't look at # tags_exclude return False if tags_exclude is not None: for tag_exclude in tags_exclude: if re.match(tag_exclude, candidate): return False return True # Both tags and tags_exclude are None, so # tag must be synced return True def process_sync_tasks(self): twenty_four_hours = 86400 # 60 * 60 * 24 is_deep_sync = self._is_deep_sync(interval=twenty_four_hours) summary = self.process_repos_query() sync_tasks = defaultdict(list) for org_key, data in summary.items(): org = org_key.org_name for item in data: push_creds = self.push_creds[org_key].split(':') image = Image(f'{item["server_url"]}/{org}/{item["name"]}', username=push_creds[0], password=push_creds[1], response_cache=self.response_cache) mirror_url = item['mirror']['url'] username = None password = None mirror_creds = None if item['mirror']['pullCredentials'] is not None: pull_credentials = item['mirror']['pullCredentials'] raw_data = self.secret_reader.read_all(pull_credentials) username = raw_data["user"] password = raw_data["token"] mirror_creds = f'{username}:{password}' image_mirror = Image(mirror_url, username=username, password=password, response_cache=self.response_cache) tags = item['mirror'].get('tags') tags_exclude = item['mirror'].get('tagsExclude') for tag in image_mirror: if not self.sync_tag(tags=tags, tags_exclude=tags_exclude, candidate=tag): continue upstream = image_mirror[tag] downstream = image[tag] if tag not in image: _LOG.debug('Image %s and mirror %s are out off sync', downstream, upstream) task = { 'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream) } sync_tasks[org_key].append(task) continue # Deep (slow) check only in non dry-run mode if self.dry_run: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue # Deep (slow) check only from time to time if not is_deep_sync: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue try: if downstream == upstream: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue except ImageComparisonError as details: _LOG.error('[%s]', details) continue _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) sync_tasks[org_key].append({ 'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream) }) return sync_tasks def _is_deep_sync(self, interval): control_file_name = 'qontract-reconcile-quay-mirror.timestamp' control_file_path = os.path.join(tempfile.gettempdir(), control_file_name) try: with open(control_file_path, 'r') as file_obj: last_deep_sync = float(file_obj.read()) except FileNotFoundError: self._record_timestamp(control_file_path) return True next_deep_sync = last_deep_sync + interval if time.time() >= next_deep_sync: self._record_timestamp(control_file_path) return True return False @staticmethod def _record_timestamp(path): with open(path, 'w') as file_object: file_object.write(str(time.time())) def _get_push_creds(self): result = self.gqlapi.query(self.QUAY_ORG_CATALOG_QUERY) creds = {} for org_data in result['quay_orgs']: push_secret = org_data['pushCredentials'] if push_secret is None: continue raw_data = self.secret_reader.read_all(push_secret) org = org_data['name'] instance = org_data['instance']['name'] org_key = OrgKey(instance, org) creds[org_key] = f'{raw_data["user"]}:{raw_data["token"]}' return creds
class DashdotdbBase: def __init__(self, dry_run, thread_pool_size, marker, scope): self.dry_run = dry_run self.thread_pool_size = thread_pool_size self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.secret_content = self.secret_reader.read_all( {'path': DASHDOTDB_SECRET}) self.dashdotdb_url = self.secret_content['url'] self.dashdotdb_user = self.secret_content['username'] self.dashdotdb_pass = self.secret_content['password'] self.logmarker = marker self.scope = scope def _get_token(self): if self.dry_run: return None params = {'scope': self.scope} endpoint = (f'{self.dashdotdb_url}/api/v1/' f'token') response = requests.get(url=endpoint, params=params, auth=(self.dashdotdb_user, self.dashdotdb_pass), timeout=(5, 120)) try: response.raise_for_status() except requests.exceptions.RequestException as details: LOG.error('%s error retrieving token for %s data: %s', self.logmarker, self.scope, details) return None self.dashdotdb_token = response.text.replace('"', '').strip() def _close_token(self): if self.dry_run: return None params = {'scope': self.scope} endpoint = (f'{self.dashdotdb_url}/api/v1/' f'token/{self.dashdotdb_token}') response = requests.delete(url=endpoint, params=params, auth=(self.dashdotdb_user, self.dashdotdb_pass), timeout=(5, 120)) try: response.raise_for_status() except requests.exceptions.RequestException as details: LOG.error('%s error closing token for %s data: %s', self.logmarker, self.scope, details) def _do_post(self, endpoint, data, timeout=(5, 120)): return requests.post(url=endpoint, json=data, headers={"X-Auth": self.dashdotdb_token}, auth=(self.dashdotdb_user, self.dashdotdb_pass), timeout=timeout) def _promget(self, url, params, token=None, ssl_verify=True, uri='api/v1/query'): url = urljoin((f'{url}'), uri) LOG.debug('%s Fetching prom payload from %s?%s', self.logmarker, url, params) headers = { "accept": "application/json", } if token: headers["Authorization"] = (f"Bearer {token}") response = requests.get(url, params=params, headers=headers, verify=ssl_verify, timeout=(5, 120)) response.raise_for_status() response = response.json() # TODO ensure len response == 1 # return response['data']['result'] return response def _get_automationtoken(self, tokenpath): autotoken_reader = SecretReader(settings=self.settings) token = autotoken_reader.read(tokenpath) return token
class SaasHerder(): """Wrapper around SaaS deployment actions.""" def __init__(self, saas_files, thread_pool_size, gitlab, integration, integration_version, settings, jenkins_map=None, accounts=None, validate=False): self.saas_files = saas_files if validate: self._validate_saas_files() if not self.valid: return self.thread_pool_size = thread_pool_size self.gitlab = gitlab self.integration = integration self.integration_version = integration_version self.settings = settings self.secret_reader = SecretReader(settings=settings) self.namespaces = self._collect_namespaces() self.jenkins_map = jenkins_map # each namespace is in fact a target, # so we can use it to calculate. divisor = len(self.namespaces) or 1 self.available_thread_pool_size = \ threaded.estimate_available_thread_pool_size( self.thread_pool_size, divisor) # if called by a single saas file,it may # specify that it manages resources exclusively. self.take_over = self._get_saas_file_attribute('takeover') self.compare = self._get_saas_file_attribute('compare') self.publish_job_logs = self._get_saas_file_attribute('publishJobLogs') if accounts: self._initiate_state(accounts) def _get_saas_file_attribute(self, attribute): return len(self.saas_files) == 1 and self.saas_files[0].get(attribute) def _validate_saas_files(self): self.valid = True saas_file_name_path_map = {} saas_file_promotion_publish_channels = [] for saas_file in self.saas_files: saas_file_name = saas_file['name'] saas_file_path = saas_file['path'] saas_file_name_path_map.setdefault(saas_file_name, []) saas_file_name_path_map[saas_file_name].append(saas_file_path) saas_file_owners = [ u['org_username'] for r in saas_file['roles'] for u in r['users'] ] if not saas_file_owners: msg = 'saas file {} has no owners: {}' logging.error(msg.format(saas_file_name, saas_file_path)) self.valid = False for resource_template in saas_file['resourceTemplates']: resource_template_name = resource_template['name'] for target in resource_template['targets']: # promotion publish channels promotion = target.get('promotion') if promotion: publish = promotion.get('publish') if publish: saas_file_promotion_publish_channels.extend( publish) # validate target parameters target_parameters = target['parameters'] if not target_parameters: continue target_parameters = json.loads(target_parameters) target_namespace = target['namespace'] namespace_name = target_namespace['name'] cluster_name = target_namespace['cluster']['name'] environment = target_namespace['environment'] environment_name = environment['name'] environment_parameters = environment['parameters'] if not environment_parameters: continue environment_parameters = \ json.loads(environment_parameters) msg = \ f'[{saas_file_name}/{resource_template_name}] ' + \ 'parameter found in target ' + \ f'{cluster_name}/{namespace_name} ' + \ f'should be reused from env {environment_name}' for t_key, t_value in target_parameters.items(): if not isinstance(t_value, str): continue for e_key, e_value in environment_parameters.items(): if not isinstance(e_value, str): continue if '.' not in e_value: continue if e_value not in t_value: continue if t_key == e_key and t_value == e_value: details = \ f'consider removing {t_key}' else: replacement = t_value.replace( e_value, '${' + e_key + '}') details = \ f'target: \"{t_key}: {t_value}\". ' + \ f'env: \"{e_key}: {e_value}\". ' + \ f'consider \"{t_key}: {replacement}\"' logging.warning(f'{msg}: {details}') # saas file name duplicates duplicates = { saas_file_name: saas_file_paths for saas_file_name, saas_file_paths in saas_file_name_path_map.items() if len(saas_file_paths) > 1 } if duplicates: self.valid = False msg = 'saas file name {} is not unique: {}' for saas_file_name, saas_file_paths in duplicates.items(): logging.error(msg.format(saas_file_name, saas_file_paths)) # promotion publish channel duplicates duplicates = [ p for p in saas_file_promotion_publish_channels if saas_file_promotion_publish_channels.count(p) > 1 ] if duplicates: self.valid = False msg = 'saas file promotion publish channel is not unique: {}' for duplicate in duplicates: logging.error(msg.format(duplicate)) def _collect_namespaces(self): # namespaces may appear more then once in the result namespaces = [] for saas_file in self.saas_files: managed_resource_types = saas_file['managedResourceTypes'] resource_templates = saas_file['resourceTemplates'] for rt in resource_templates: targets = rt['targets'] for target in targets: namespace = target['namespace'] if target.get('disable'): logging.debug( f"[{saas_file['name']}/{rt['name']}] target " + f"{namespace['cluster']['name']}/" + f"{namespace['name']} is disabled.") continue # managedResourceTypes is defined per saas_file # add it to each namespace in the current saas_file namespace['managedResourceTypes'] = managed_resource_types namespaces.append(namespace) return namespaces def _initiate_state(self, accounts): self.state = State(integration=self.integration, accounts=accounts, settings=self.settings) @staticmethod def _collect_parameters(container): parameters = container.get('parameters') or {} if isinstance(parameters, str): parameters = json.loads(parameters) # adjust Python's True/False for k, v in parameters.items(): if v is True: parameters[k] = 'true' elif v is False: parameters[k] = 'false' elif any([isinstance(v, t) for t in [dict, list, tuple]]): parameters[k] = json.dumps(v) return parameters @staticmethod def _get_file_contents_github(repo, path, commit_sha): try: f = repo.get_contents(path, commit_sha) return f.decoded_content except GithubException as e: # slightly copied with love from # https://github.com/PyGithub/PyGithub/issues/661 errors = e.data['errors'] # example errors dict that we are looking for # { # 'message': '<text>', # 'errors': [{ # 'resource': 'Blob', # 'field': 'data', # 'code': 'too_large' # }], # 'documentation_url': '<url>' # } for error in errors: if error['code'] == 'too_large': # get large files tree = repo.get_git_tree(commit_sha, recursive='/' in path).tree for x in tree: if x.path != path.lstrip('/'): continue blob = repo.get_git_blob(x.sha) return base64.b64decode(blob.content).decode("utf8") raise e @retry() def _get_file_contents(self, options): url = options['url'] path = options['path'] ref = options['ref'] github = options['github'] html_url = f"{url}/blob/{ref}{path}" commit_sha = self._get_commit_sha(options) content = None if 'github' in url: repo_name = url.rstrip("/").replace('https://github.com/', '') repo = github.get_repo(repo_name) content = self._get_file_contents_github(repo, path, commit_sha) elif 'gitlab' in url: if not self.gitlab: raise Exception('gitlab is not initialized') project = self.gitlab.get_project(url) f = project.files.get(file_path=path.lstrip('/'), ref=commit_sha) content = f.decode() return yaml.safe_load(content), html_url, commit_sha @retry() def _get_directory_contents(self, options): url = options['url'] path = options['path'] ref = options['ref'] github = options['github'] html_url = f"{url}/tree/{ref}{path}" commit_sha = self._get_commit_sha(options) resources = [] if 'github' in url: repo_name = url.rstrip("/").replace('https://github.com/', '') repo = github.get_repo(repo_name) for f in repo.get_contents(path, commit_sha): file_path = os.path.join(path, f.name) file_contents_decoded = \ self._get_file_contents_github( repo, file_path, commit_sha) resource = yaml.safe_load(file_contents_decoded) resources.append(resource) elif 'gitlab' in url: if not self.gitlab: raise Exception('gitlab is not initialized') project = self.gitlab.get_project(url) for f in project.repository_tree(path=path.lstrip('/'), ref=commit_sha, all=True): file_contents = \ project.files.get(file_path=f['path'], ref=commit_sha) resource = yaml.safe_load(file_contents.decode()) resources.append(resource) return resources, html_url, commit_sha @retry() def _get_commit_sha(self, options): url = options['url'] ref = options['ref'] github = options['github'] hash_length = options.get('hash_length') commit_sha = '' if 'github' in url: repo_name = url.rstrip("/").replace('https://github.com/', '') repo = github.get_repo(repo_name) commit = repo.get_commit(sha=ref) commit_sha = commit.sha elif 'gitlab' in url: if not self.gitlab: raise Exception('gitlab is not initialized') project = self.gitlab.get_project(url) commits = project.commits.list(ref_name=ref) commit_sha = commits[0].id if hash_length: return commit_sha[:hash_length] return commit_sha @staticmethod def _get_cluster_and_namespace(target): cluster = target['namespace']['cluster']['name'] namespace = target['namespace']['name'] return cluster, namespace @staticmethod def _additional_resource_process(resources, html_url): for resource in resources: # add a definition annotation to each PrometheusRule rule if resource['kind'] == 'PrometheusRule': try: groups = resource['spec']['groups'] for group in groups: rules = group['rules'] for rule in rules: annotations = rule.get('annotations') if not annotations: continue rule['annotations']['html_url'] = html_url except Exception: logging.warning('could not add html_url annotation to' + resource['name']) @staticmethod def _parameter_value_needed(parameter_name, consolidated_parameters, template): """Is a parameter named in the template but unspecified? NOTE: This is currently "parameter *named* and absent" -- i.e. we don't care about `required: true`. This is for backward compatibility. :param parameter_name: The name (key) of the parameter. :param consolidated_parameters: Dict of parameters already specified/ calculated. :param template: The template file in dict form. :return bool: True if the named parameter is named in the template, but not already present in consolidated_parameters. """ if parameter_name in consolidated_parameters: return False for template_parameter in template.get("parameters", {}): if template_parameter["name"] == parameter_name: return True return False def _process_template(self, options): saas_file_name = options['saas_file_name'] resource_template_name = options['resource_template_name'] image_auth = options['image_auth'] url = options['url'] path = options['path'] provider = options['provider'] target = options['target'] github = options['github'] target_ref = target['ref'] target_promotion = target.get('promotion') or {} resources = None html_url = None commit_sha = None if provider == 'openshift-template': hash_length = options['hash_length'] parameters = options['parameters'] environment = target['namespace']['environment'] environment_parameters = self._collect_parameters(environment) target_parameters = self._collect_parameters(target) consolidated_parameters = {} consolidated_parameters.update(environment_parameters) consolidated_parameters.update(parameters) consolidated_parameters.update(target_parameters) for replace_key, replace_value in consolidated_parameters.items(): if not isinstance(replace_value, str): continue replace_pattern = '${' + replace_key + '}' for k, v in consolidated_parameters.items(): if not isinstance(v, str): continue if replace_pattern in v: consolidated_parameters[k] = \ v.replace(replace_pattern, replace_value) get_file_contents_options = { 'url': url, 'path': path, 'ref': target_ref, 'github': github } try: template, html_url, commit_sha = \ self._get_file_contents(get_file_contents_options) except Exception as e: logging.error(f"[{url}/{path}:{target_ref}] " + f"error fetching template: {str(e)}") return None, None, None # add IMAGE_TAG only if it is unspecified image_tag = consolidated_parameters.get('IMAGE_TAG') if not image_tag: sha_substring = commit_sha[:hash_length] # IMAGE_TAG takes one of two forms: # - If saas file attribute 'use_channel_in_image_tag' is true, # it is {CHANNEL}-{SHA} # - Otherwise it is just {SHA} if self._get_saas_file_attribute("use_channel_in_image_tag"): try: channel = consolidated_parameters["CHANNEL"] except KeyError: logging.error( f"[{saas_file_name}/{resource_template_name}] " + f"{html_url}: CHANNEL is required when " + "'use_channel_in_image_tag' is true.") return None, None, None image_tag = f"{channel}-{sha_substring}" else: image_tag = sha_substring consolidated_parameters['IMAGE_TAG'] = image_tag # This relies on IMAGE_TAG already being calculated. need_repo_digest = self._parameter_value_needed( "REPO_DIGEST", consolidated_parameters, template) need_image_digest = self._parameter_value_needed( "IMAGE_DIGEST", consolidated_parameters, template) if need_repo_digest or need_image_digest: try: logging.debug("Generating REPO_DIGEST.") registry_image = consolidated_parameters["REGISTRY_IMG"] except KeyError as e: logging.error( f"[{saas_file_name}/{resource_template_name}] " + f"{html_url}: error generating REPO_DIGEST. " + "Is REGISTRY_IMG missing? " + f"{str(e)}") return None, None, None try: image_uri = f"{registry_image}:{image_tag}" img = Image(image_uri, **image_auth) if need_repo_digest: consolidated_parameters["REPO_DIGEST"] = img.url_digest if need_image_digest: consolidated_parameters["IMAGE_DIGEST"] = img.digest except (rqexc.ConnectionError, rqexc.HTTPError) as e: logging.error( f"[{saas_file_name}/{resource_template_name}] " + f"{html_url}: error generating REPO_DIGEST for " + f"{image_uri}: {str(e)}") return None, None, None oc = OC('server', 'token', local=True) try: resources = oc.process(template, consolidated_parameters) except StatusCodeError as e: logging.error( f"[{saas_file_name}/{resource_template_name}] " + f"{html_url}: error processing template: {str(e)}") elif provider == 'directory': get_directory_contents_options = { 'url': url, 'path': path, 'ref': target_ref, 'github': github } try: resources, html_url, commit_sha = \ self._get_directory_contents( get_directory_contents_options) except Exception as e: logging.error(f"[{url}/{path}:{target_ref}] " + f"error fetching directory: {str(e)}") return None, None, None else: logging.error(f"[{saas_file_name}/{resource_template_name}] " + f"unknown provider: {provider}") target_promotion['commit_sha'] = commit_sha return resources, html_url, target_promotion @staticmethod def _collect_images(resource): images = set() # resources with pod templates try: template = resource["spec"]["template"] for c in template["spec"]["containers"]: images.add(c["image"]) except KeyError: pass # init containers try: template = resource["spec"]["template"] for c in template["spec"]["initContainers"]: images.add(c["image"]) except KeyError: pass # CronJob try: template = resource["spec"]["jobTemplate"]["spec"]["template"] for c in template["spec"]["containers"]: images.add(c["image"]) except KeyError: pass # CatalogSource templates try: images.add(resource["spec"]["image"]) except KeyError: pass return images @staticmethod def _check_image(image, image_patterns, image_auth, error_prefix): error = False if image_patterns and \ not any(image.startswith(p) for p in image_patterns): error = True logging.error( f"{error_prefix} Image is not in imagePatterns: {image}") try: valid = Image(image, **image_auth) if not valid: error = True logging.error(f"{error_prefix} Image does not exist: {image}") except Exception as e: error = True logging.error(f"{error_prefix} Image is invalid: {image}. " + f"details: {str(e)}") return error def _check_images(self, options): saas_file_name = options['saas_file_name'] resource_template_name = options['resource_template_name'] html_url = options['html_url'] resources = options['resources'] image_auth = options['image_auth'] image_patterns = options['image_patterns'] error_prefix = \ f"[{saas_file_name}/{resource_template_name}] {html_url}:" images_list = threaded.run(self._collect_images, resources, self.available_thread_pool_size) images = {item for sublist in images_list for item in sublist} if not images: return False # no errors errors = threaded.run(self._check_image, images, self.available_thread_pool_size, image_patterns=image_patterns, image_auth=image_auth, error_prefix=error_prefix) error = True in errors return error def _initiate_github(self, saas_file): auth = saas_file.get('authentication') or {} auth_code = auth.get('code') or {} if auth_code: token = self.secret_reader.read(auth_code) else: # use the app-sre token by default default_org_name = 'app-sre' config = get_config(desired_org_name=default_org_name) token = config['github'][default_org_name]['token'] base_url = os.environ.get('GITHUB_API', 'https://api.github.com') # This is a threaded world. Let's define a big # connections pool to live in that world # (this avoids the warning "Connection pool is # full, discarding connection: api.github.com") pool_size = 100 return Github(token, base_url=base_url, pool_size=pool_size) def _initiate_image_auth(self, saas_file): """ This function initiates a dict required for image authentication. This dict will be used as kwargs for sertoolbox's Image. The image authentication secret specified in the saas file must contain the 'user' and 'token' keys, and may optionally contain a 'url' key specifying the image registry url to be passed to check if an image should be checked using these credentials. The function returns the keys extracted from the secret in the structure expected by sretoolbox's Image: 'user' --> 'username' 'token' --> 'password' 'url' --> 'auth_server' (optional) """ auth = saas_file.get('authentication') if not auth: return {} auth_image_secret = auth.get('image') if not auth_image_secret: return {} creds = self.secret_reader.read_all(auth_image_secret) required_keys = ['user', 'token'] ok = all(k in creds.keys() for k in required_keys) if not ok: logging.warning( "the specified image authentication secret " + f"found in path {auth_image_secret['path']} " + f"does not contain all required keys: {required_keys}") return {} image_auth = {'username': creds['user'], 'password': creds['token']} url = creds.get('url') if url: image_auth['auth_server']: url return image_auth def populate_desired_state(self, ri): results = threaded.run(self.init_populate_desired_state_specs, self.saas_files, self.thread_pool_size) desired_state_specs = \ [item for sublist in results for item in sublist] promotions = threaded.run(self.populate_desired_state_saas_file, desired_state_specs, self.thread_pool_size, ri=ri) self.promotions = promotions def init_populate_desired_state_specs(self, saas_file): specs = [] saas_file_name = saas_file['name'] github = self._initiate_github(saas_file) image_auth = self._initiate_image_auth(saas_file) instance_name = saas_file['instance']['name'] managed_resource_types = saas_file['managedResourceTypes'] image_patterns = saas_file['imagePatterns'] resource_templates = saas_file['resourceTemplates'] saas_file_parameters = self._collect_parameters(saas_file) # iterate over resource templates (multiple per saas_file) for rt in resource_templates: rt_name = rt['name'] url = rt['url'] path = rt['path'] provider = rt.get('provider') or 'openshift-template' hash_length = rt.get('hash_length') or self.settings['hashLength'] parameters = self._collect_parameters(rt) consolidated_parameters = {} consolidated_parameters.update(saas_file_parameters) consolidated_parameters.update(parameters) # iterate over targets (each target is a namespace) for target in rt['targets']: if target.get('disable'): # a warning is logged during SaasHerder initiation continue cluster, namespace = \ self._get_cluster_and_namespace(target) process_template_options = { 'saas_file_name': saas_file_name, 'resource_template_name': rt_name, 'image_auth': image_auth, 'url': url, 'path': path, 'provider': provider, 'hash_length': hash_length, 'target': target, 'parameters': consolidated_parameters, 'github': github } check_images_options_base = { 'saas_file_name': saas_file_name, 'resource_template_name': rt_name, 'image_auth': image_auth, 'image_patterns': image_patterns } spec = { 'saas_file_name': saas_file_name, 'cluster': cluster, 'namespace': namespace, 'managed_resource_types': managed_resource_types, 'process_template_options': process_template_options, 'check_images_options_base': check_images_options_base, 'instance_name': instance_name, 'upstream': target.get('upstream') } specs.append(spec) return specs def populate_desired_state_saas_file(self, spec, ri): saas_file_name = spec['saas_file_name'] cluster = spec['cluster'] namespace = spec['namespace'] managed_resource_types = spec['managed_resource_types'] process_template_options = spec['process_template_options'] check_images_options_base = spec['check_images_options_base'] instance_name = spec['instance_name'] upstream = spec['upstream'] resources, html_url, promotion = \ self._process_template(process_template_options) if resources is None: ri.register_error() return # filter resources resources = [ resource for resource in resources if isinstance(resource, dict) and resource['kind'] in managed_resource_types ] # additional processing of resources self._additional_resource_process(resources, html_url) # check images skip_check_images = upstream and self.jenkins_map and \ self.jenkins_map[instance_name].is_job_running(upstream) if skip_check_images: logging.warning("skipping check_image since " + f"upstream job {upstream} is running") else: check_images_options = { 'html_url': html_url, 'resources': resources } check_images_options.update(check_images_options_base) image_error = self._check_images(check_images_options) if image_error: ri.register_error() return # add desired resources for resource in resources: resource_kind = resource['kind'] resource_name = resource['metadata']['name'] oc_resource = OR(resource, self.integration, self.integration_version, caller_name=saas_file_name, error_details=html_url) ri.add_desired(cluster, namespace, resource_kind, resource_name, oc_resource) return promotion def get_moving_commits_diff(self, dry_run): results = threaded.run(self.get_moving_commits_diff_saas_file, self.saas_files, self.thread_pool_size, dry_run=dry_run) return [item for sublist in results for item in sublist] def get_moving_commits_diff_saas_file(self, saas_file, dry_run): saas_file_name = saas_file['name'] instace_name = saas_file['instance']['name'] github = self._initiate_github(saas_file) trigger_specs = [] for rt in saas_file['resourceTemplates']: rt_name = rt['name'] url = rt['url'] for target in rt['targets']: # don't trigger if there is a linked upstream job if target.get('upstream'): continue ref = target['ref'] get_commit_sha_options = { 'url': url, 'ref': ref, 'github': github } desired_commit_sha = \ self._get_commit_sha(get_commit_sha_options) # don't trigger on refs which are commit shas if ref == desired_commit_sha: continue namespace = target['namespace'] cluster_name = namespace['cluster']['name'] namespace_name = namespace['name'] env_name = namespace['environment']['name'] key = f"{saas_file_name}/{rt_name}/{cluster_name}/" + \ f"{namespace_name}/{env_name}/{ref}" current_commit_sha = self.state.get(key, None) # skip if there is no change in commit sha if current_commit_sha == desired_commit_sha: continue # don't trigger if this is the first time # this target is being deployed. # that will be taken care of by # openshift-saas-deploy-trigger-configs if current_commit_sha is None: # store the value to take over from now on if not dry_run: self.state.add(key, value=desired_commit_sha) continue # we finally found something we want to trigger on! job_spec = { 'saas_file_name': saas_file_name, 'env_name': env_name, 'instance_name': instace_name, 'rt_name': rt_name, 'cluster_name': cluster_name, 'namespace_name': namespace_name, 'ref': ref, 'commit_sha': desired_commit_sha } trigger_specs.append(job_spec) return trigger_specs def update_moving_commit(self, job_spec): saas_file_name = job_spec['saas_file_name'] env_name = job_spec['env_name'] rt_name = job_spec['rt_name'] cluster_name = job_spec['cluster_name'] namespace_name = job_spec['namespace_name'] ref = job_spec['ref'] commit_sha = job_spec['commit_sha'] key = f"{saas_file_name}/{rt_name}/{cluster_name}/" + \ f"{namespace_name}/{env_name}/{ref}" self.state.add(key, value=commit_sha, force=True) def get_configs_diff(self): results = threaded.run(self.get_configs_diff_saas_file, self.saas_files, self.thread_pool_size) return [item for sublist in results for item in sublist] def get_configs_diff_saas_file(self, saas_file): saas_file_name = saas_file['name'] saas_file_parameters = saas_file.get('parameters') saas_file_managed_resource_types = saas_file['managedResourceTypes'] instace_name = saas_file['instance']['name'] trigger_specs = [] for rt in saas_file['resourceTemplates']: rt_name = rt['name'] url = rt['url'] path = rt['path'] rt_parameters = rt.get('parameters') for desired_target_config in rt['targets']: namespace = desired_target_config['namespace'] cluster_name = namespace['cluster']['name'] namespace_name = namespace['name'] env_name = namespace['environment']['name'] desired_target_config['namespace'] = \ self.sanitize_namespace(namespace) # add parent parameters to target config desired_target_config['saas_file_parameters'] = \ saas_file_parameters # add managed resource types to target config desired_target_config['saas_file_managed_resource_types'] = \ saas_file_managed_resource_types desired_target_config['url'] = url desired_target_config['path'] = path desired_target_config['rt_parameters'] = rt_parameters # get current target config from state key = f"{saas_file_name}/{rt_name}/{cluster_name}/" + \ f"{namespace_name}/{env_name}" current_target_config = self.state.get(key, None) # skip if there is no change in target configuration if current_target_config == desired_target_config: continue job_spec = { 'saas_file_name': saas_file_name, 'env_name': env_name, 'instance_name': instace_name, 'rt_name': rt_name, 'cluster_name': cluster_name, 'namespace_name': namespace_name, 'target_config': desired_target_config } trigger_specs.append(job_spec) return trigger_specs @staticmethod def sanitize_namespace(namespace): """Only keep fields that should trigger a new job.""" new_job_fields = { 'namespace': ['name', 'cluster', 'app'], 'cluster': ['name', 'serverUrl'], 'app': ['name'] } namespace = { k: v for k, v in namespace.items() if k in new_job_fields['namespace'] } cluster = namespace['cluster'] namespace['cluster'] = { k: v for k, v in cluster.items() if k in new_job_fields['cluster'] } app = namespace['app'] namespace['app'] = { k: v for k, v in app.items() if k in new_job_fields['app'] } return namespace def update_config(self, job_spec): saas_file_name = job_spec['saas_file_name'] env_name = job_spec['env_name'] rt_name = job_spec['rt_name'] cluster_name = job_spec['cluster_name'] namespace_name = job_spec['namespace_name'] target_config = job_spec['target_config'] key = f"{saas_file_name}/{rt_name}/{cluster_name}/" + \ f"{namespace_name}/{env_name}" self.state.add(key, value=target_config, force=True) def validate_promotions(self): """ If there were promotion sections in the participating saas files validate that the conditions are met. """ for item in self.promotions: if item is None: continue # validate that the commit sha being promoted # was succesfully published to the subscribed channel(s) commit_sha = item['commit_sha'] subscribe = item.get('subscribe') if subscribe: for channel in subscribe: state_key = f"promotions/{channel}/{commit_sha}" value = self.state.get(state_key, None) success = value.get('success') if not success: logging.error( f'Commit {commit_sha} was not ' + f'published with success to channel {channel}') return False return True def publish_promotions(self, success, saas_files, mr_cli): """ If there were promotion sections in the participating saas files publish the results for future promotion validations. """ subscribe_saas_file_path_map = \ self._get_subscribe_saas_file_path_map(saas_files, auto_only=True) trigger_promotion = False for item in self.promotions: commit_sha = item['commit_sha'] publish = item.get('publish') if publish: all_subscribed_saas_file_paths = set() for channel in publish: # publish to state to pass promotion gate state_key = f"promotions/{channel}/{commit_sha}" value = {'success': success} self.state.add(state_key, value, force=True) logging.info( f'Commit {commit_sha} was published ' + f'with success {success} to channel {channel}') # collect data to trigger promotion subscribed_saas_file_paths = \ subscribe_saas_file_path_map.get(channel) if subscribed_saas_file_paths: all_subscribed_saas_file_paths.update( subscribed_saas_file_paths) item['saas_file_paths'] = list(all_subscribed_saas_file_paths) if all_subscribed_saas_file_paths: trigger_promotion = True if trigger_promotion: mr = AutoPromoter(self.promotions) mr.submit(cli=mr_cli) @staticmethod def _get_subscribe_saas_file_path_map(saas_files, auto_only=False): """ Returns a dict with subscribe channels as keys and a list of paths of saas files containing these channels. """ subscribe_saas_file_path_map = {} for saas_file in saas_files: saas_file_path = 'data' + saas_file['path'] for rt in saas_file['resourceTemplates']: for target in rt['targets']: target_promotion = target.get('promotion') if not target_promotion: continue target_auto = target_promotion.get('auto') if auto_only and not target_auto: continue subscribe = target_promotion.get('subscribe') if not subscribe: continue for channel in subscribe: subscribe_saas_file_path_map.setdefault(channel, set()) subscribe_saas_file_path_map[channel].add( saas_file_path) return subscribe_saas_file_path_map
def get_tf_secrets(account: Account, secret_reader: SecretReader) -> Tuple[str, Dict]: account_name = account["name"] automation_token = account["automationToken"] secret = secret_reader.read_all(automation_token) return (account_name, secret)