def _import_autoscaling_region(proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'import autoscaling in {region}') if resource_gate(spec, 'AutoScalingGroup'): yield from _import_autoscaling_groups(proxy) if resource_gate(spec, 'LaunchConfiguration'): yield from _import_launch_configurations(proxy)
def _import_elbv2_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: _log.info(f'importing elbv2 {region}') if resource_gate(spec, 'LoadBalancer'): yield from _import_loadbalancers(proxy, spec) if resource_gate(spec, 'TargetGroup'): yield from _import_target_groups(proxy)
def _import_ecs_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: if resource_gate(spec, 'Cluster'): _log.info(f'importing ecs clusters') yield from _import_clusters(proxy, region, spec) if resource_gate(spec, 'TaskDefinition'): yield from _import_task_definitions(proxy)
def _import_lambda_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: if resource_gate(spec, 'Function'): _log.info(f'Importing functions in {region}') yield from _import_functions(proxy, spec) if resource_gate(spec, 'LayerVersion'): yield from _import_layers(proxy)
def _import_sns_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: _log.info(f'importing SNS in {region}') if resource_gate(spec, 'Topic'): yield from _import_topics(proxy) if resource_gate(spec, 'Subscription'): yield from _import_subscriptions(proxy)
def _import_dynamodb_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: if resource_gate(spec, 'Table'): _log.info(f'importing dynamodb tables {region}') yield from _import_tables(proxy, region) if resource_gate(spec, 'Backup'): _log.info(f'importing dynamodb backups {region}') yield from _import_backups(proxy, region)
def _import_clusters(proxy: ServiceProxy, region: str, spec: ServiceSpec): clusters_resp = proxy.list('list_clusters') if clusters_resp is not None: cluster_arns = clusters_resp[1].get('clusterArns', []) for cluster_arn in cluster_arns: yield 'Cluster', _import_cluster(proxy, cluster_arn) if resource_gate(spec, 'Service'): yield from _import_services(proxy, cluster_arn) if resource_gate(spec, 'Task'): yield from _import_tasks(proxy, cluster_arn)
def _import_elasticbeanstalk_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: _log.info(f'importing ElasticBeanstalk {region}') if resource_gate(spec, 'Application'): yield from _import_applications(proxy) if resource_gate(spec, 'ApplicationVersion'): yield from _import_application_versions(proxy) if resource_gate(spec, 'Environment'): yield from _import_environments(proxy)
def _import_rds_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: if resource_gate(spec, 'DBCluster'): yield from _import_db_clusters(proxy) if resource_gate(spec, 'DBInstance'): yield from _import_db_instances(proxy) if resource_gate(spec, 'DBSnapshot'): yield from _import_db_snapshots(proxy) if resource_gate(spec, 'DBClusterSnapshot'): yield from _import_db_cluster_snapshots(proxy)
def _import_route53(proxy: ServiceProxy, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: if resource_gate(spec, 'HostzedZone'): zones_resp = proxy.list('list_hosted_zones') if zones_resp is not None: for zone in zones_resp[1]['HostedZones']: yield 'HostedZone', _import_hosted_zone(proxy, zone)
def _import_clusters(proxy: ServiceProxy, region: str, spec: ServiceSpec): clusters_resp = proxy.list('list_clusters') if clusters_resp is not None: cluster_names = clusters_resp[1].get('clusters', []) for cluster_name in cluster_names: yield 'Cluster', _import_cluster(proxy, cluster_name) if resource_gate(spec, 'Nodegroup'): yield from _import_nodegroups(proxy, cluster_name)
def _import_loadbalancers(proxy: ServiceProxy, spec: ServiceSpec): lbs_resp = proxy.list('describe_load_balancers') if lbs_resp is not None: lbs = lbs_resp[1].get('LoadBalancers', []) for lb in lbs: yield 'LoadBalancer', _import_loadbalancer(proxy, lb) if resource_gate(spec, 'Listener'): yield from _import_listeners(proxy, lb['LoadBalancerArn'])
def _import_elb_region(proxy: ServiceProxy, region: str, spec: ServiceSpec): if resource_gate(spec, 'LoadBalancer'): _log.info(f'Importing {region} load balancers') result = proxy.list('describe_load_balancers') if result is not None: elbs = result[1] for elb in elbs.get('LoadBalancerDescriptions', []): if not _is_v2(elb['LoadBalancerName']): yield 'LoadBalancer', import_elb(proxy, elb)
def _import_function(proxy: ServiceProxy, function: Dict, spec: ServiceSpec): name = function['FunctionName'] arn = function['FunctionArn'] if resource_gate(spec, 'FunctionVersion'): versions_resp = proxy.list('list_versions_by_function', FunctionName=name) if versions_resp is not None: versions = versions_resp[1]['Versions'] for version in versions: version['ParentFunctionArn'] = arn version['Policy'] = _get_policy(proxy, version['FunctionArn']) yield 'FunctionVersion', version if resource_gate(spec, 'Alias'): aliases_resp = proxy.list('list_aliases', FunctionName=name) if aliases_resp is not None: aliases = aliases_resp[1]['Aliases'] for alias in aliases: alias['FunctionArn'] = arn alias['Policy'] = _get_policy(proxy, alias['AliasArn']) yield 'Alias', alias
def _import_roles(proxy: ServiceProxy, spec: ServiceSpec): if resource_gate(spec, 'role'): roles = _unpack(proxy.list('list_roles')) for role in roles['Roles']: role_data = proxy.get('get_role', RoleName=role['RoleName'])['Role'] name = role_data['RoleName'] for attr, op in _ROLE_ATTRS.items(): op_result = _unpack(proxy.list(op, RoleName=name)) role_data[attr] = op_result.get(attr) role_data['PolicyList'] = _fetch_inline_policies( proxy, 'role', name) yield 'role', role_data
def _import_groups(proxy: ServiceProxy, spec: ServiceSpec): if resource_gate(spec, 'group'): groups = _unpack(proxy.list('list_groups')) for group in groups.get('Groups'): group_data = group.copy() name = group_data['GroupName'] for attr, op in _GROUP_ATTRS.items(): op_result = proxy.list(op, GroupName=name) if op_result is not None: group_data[attr] = op_result[1][attr] group_data['PolicyList'] = _fetch_inline_policies( proxy, 'group', name) yield 'group', group_data
def _import_ec2_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: for resource in proxy.resource_names(): _log.info(f'importing {resource}') op_name = proxy._impl._client._PY_TO_OP_NAME[resource] op_resource = op_name[len('Describe'):] if op_resource in RESOURCES: if resource_gate(spec, op_resource): result = proxy.list(resource) if result is not None: if resource == 'describe_instances': _add_user_data(proxy, result[1]) elif resource == 'describe_snapshots': _add_launch_permissions(proxy, result[1]) elif resource == 'describe_images': _add_image_attributes(proxy, result[1]) elif resource == 'describe_security_groups': _add_security_group_references(proxy, result[1]) yield result[0], result[1] _log.info(f'done with {resource}') if resource_gate(spec, 'Defaults'): yield from _synthesize_defaults(proxy, region)
def _import_credential_report(proxy: ServiceProxy, spec: ServiceSpec): if resource_gate(spec, 'CredentialReport'): # Kick off the report started = False init_attempts = 0 while not started: try: resp = proxy.get('generate_credential_report') started = resp.get('State') in ('STARTED', 'COMPLETE') except ClientError as e: code = e.response.get('Error', {}).get('Code') is_throttled = code == 'Throttling' if not is_throttled: _log.error('credential report error', exc_info=e) raise # wait and try again? init_attempts += 1 if init_attempts >= 3: _log.error('credential report error', exc_info=e) raise GFError('Failed to generate credential report') time.sleep(0.1) except SSLError: # wait and try again? init_attempts += 1 if init_attempts >= 3: raise GFError( 'Failed to generate credential report, SSL Error') time.sleep(0.1) attempts = 0 report = None while attempts < 20 and report is None: try: report = proxy.get('get_credential_report') except ClientError as e: code = e.response.get('Error', {}).get('Code') if code == 'ReportInProgress': attempts += 1 time.sleep(1) else: _log.error('credenetial report fetch error', exc_info=e) raise if report is None: raise GFError('Failed to fetch credential report') decoded = report['Content'].decode('utf-8') reader = csv.DictReader(StringIO(decoded)) for row in reader: processed = _post_process_report_row(row) yield 'CredentialReport', processed
def _import_policies(proxy: ServiceProxy, spec: ServiceSpec): if resource_gate(spec, 'policy'): results = proxy.list('list_policies', Scope='Local') if results is not None: policies = results[1] for policy in policies['Policies']: policy_data = policy.copy() arn = policy_data['Arn'] versions_result = _unpack( proxy.list('list_policy_versions', PolicyArn=arn)) versions = [] for version in versions_result.get('Versions', []): full_version = proxy.get('get_policy_version', PolicyArn=arn, VersionId=version['VersionId']) versions.append(full_version['PolicyVersion']) policy_data['Versions'] = versions op_result = proxy.list('list_entities_for_policy', PolicyArn=arn) if op_result is not None: for attr in ['PolicyGroups', 'PolicyUsers', 'PolicyRoles']: policy_data[attr] = op_result[1][attr] yield 'policy', policy_data # TODO: fix this cut + paste aws_policies = proxy.list('list_policies', Scope='AWS', OnlyAttached=True) if aws_policies is not None: policies = aws_policies[1] for policy in policies['Policies']: policy_data = policy.copy() arn = policy_data['Arn'] versions_result = _unpack( proxy.list('list_policy_versions', PolicyArn=arn)) versions = [] for version in versions_result.get('Versions', []): full_version = proxy.get('get_policy_version', PolicyArn=arn, VersionId=version['VersionId']) versions.append(full_version['PolicyVersion']) policy_data['Versions'] = versions op_result = proxy.list('list_entities_for_policy', PolicyArn=arn) if op_result is not None: for attr in ['PolicyGroups', 'PolicyUsers', 'PolicyRoles']: policy_data[attr] = op_result[1][attr] yield 'policy', policy_data
def _import_users(proxy: ServiceProxy, spec: ServiceSpec): if resource_gate(spec, 'user'): users = _unpack(proxy.list('list_users')) for user in users['Users']: user_data = user.copy() name = user_data['UserName'] for attr, op_desc in _USER_ATTRS.items(): if isinstance(op_desc, str): op = op_desc field = attr else: op = op_desc['op'] field = op_desc['key'] op_result = proxy.list(op, UserName=name) if op_result is not None: user_data[attr] = op_result[1][field] user_data['PolicyList'] = _fetch_inline_policies( proxy, 'user', name) login_profile = proxy.get('get_login_profile', UserName=name) if login_profile is not None: login_profile = login_profile['LoginProfile'] user_data['LoginProfile'] = login_profile yield 'user', user_data
def _import_logs_region(proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: if resource_gate(spec, 'LogGroup'): _log.info(f'Import LogGroups in {region}') yield from _import_log_groups(proxy)
def _import_cloudfront(proxy: ServiceProxy, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'importing cloudfront distributions') if resource_gate(spec, 'Distribution'): yield from _import_distributions(proxy)
def _import_efs_region(proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'import efs in {region}') if resource_gate(spec, 'FileSystem'): yield from _import_filesystems(proxy)
def _import_glacier_region(proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'import glacier in {region}') if resource_gate(spec, 'Vault'): yield from _import_vaults(proxy)
def _import_acm_pca_region(proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'import acm-pca in {region}') if resource_gate(spec, 'CertificateAuthority'): yield from _import_certificate_authorities(proxy)
def _import_sqs_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: if resource_gate(spec, 'Queue'): _log.info(f'importing sqs Queues in {region}') yield from _import_queues(proxy, region)
def _import_kms_region(proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'import kms in {region}') if resource_gate(spec, 'Key'): yield from _import_keys(proxy)
def _import_cloudformation_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Iterator[Tuple[str, Any]]: _log.info(f'import cloudformation in {region}') if resource_gate(spec, 'Stack'): yield from _import_cloudformation_stacks(proxy)
def _import_ssm_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: if resource_gate(spec, 'Parameter'): _log.info(f'importing describe_parameters {region}') yield from _import_parameters(proxy, region)
def _import_cloudtrail_region( proxy: ServiceProxy, region: str, spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]: _log.info(f'importing describe_trails in {region}') if resource_gate(spec, 'Trail'): yield from _import_trails(proxy, region)