def _import_autoscaling_region(proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'import autoscaling in {region}')
if resource_gate(spec, 'AutoScalingGroup'):
yield from _import_autoscaling_groups(proxy)
if resource_gate(spec, 'LaunchConfiguration'):
yield from _import_launch_configurations(proxy)
def _import_elbv2_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
_log.info(f'importing elbv2 {region}')
if resource_gate(spec, 'LoadBalancer'):
yield from _import_loadbalancers(proxy, spec)
if resource_gate(spec, 'TargetGroup'):
yield from _import_target_groups(proxy)
def _import_ecs_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
if resource_gate(spec, 'Cluster'):
_log.info(f'importing ecs clusters')
yield from _import_clusters(proxy, region, spec)
if resource_gate(spec, 'TaskDefinition'):
yield from _import_task_definitions(proxy)
def _import_lambda_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
if resource_gate(spec, 'Function'):
_log.info(f'Importing functions in {region}')
yield from _import_functions(proxy, spec)
if resource_gate(spec, 'LayerVersion'):
yield from _import_layers(proxy)
def _import_sns_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
_log.info(f'importing SNS in {region}')
if resource_gate(spec, 'Topic'):
yield from _import_topics(proxy)
if resource_gate(spec, 'Subscription'):
yield from _import_subscriptions(proxy)
def _import_dynamodb_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
if resource_gate(spec, 'Table'):
_log.info(f'importing dynamodb tables {region}')
yield from _import_tables(proxy, region)
if resource_gate(spec, 'Backup'):
_log.info(f'importing dynamodb backups {region}')
yield from _import_backups(proxy, region)
def _import_clusters(proxy: ServiceProxy, region: str, spec: ServiceSpec):
clusters_resp = proxy.list('list_clusters')
if clusters_resp is not None:
cluster_arns = clusters_resp[1].get('clusterArns', [])
for cluster_arn in cluster_arns:
yield 'Cluster', _import_cluster(proxy, cluster_arn)
if resource_gate(spec, 'Service'):
yield from _import_services(proxy, cluster_arn)
if resource_gate(spec, 'Task'):
yield from _import_tasks(proxy, cluster_arn)
def _import_elasticbeanstalk_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
_log.info(f'importing ElasticBeanstalk {region}')
if resource_gate(spec, 'Application'):
yield from _import_applications(proxy)
if resource_gate(spec, 'ApplicationVersion'):
yield from _import_application_versions(proxy)
if resource_gate(spec, 'Environment'):
yield from _import_environments(proxy)
def _import_rds_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
if resource_gate(spec, 'DBCluster'):
yield from _import_db_clusters(proxy)
if resource_gate(spec, 'DBInstance'):
yield from _import_db_instances(proxy)
if resource_gate(spec, 'DBSnapshot'):
yield from _import_db_snapshots(proxy)
if resource_gate(spec, 'DBClusterSnapshot'):
yield from _import_db_cluster_snapshots(proxy)
def _import_route53(proxy: ServiceProxy,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
if resource_gate(spec, 'HostzedZone'):
zones_resp = proxy.list('list_hosted_zones')
if zones_resp is not None:
for zone in zones_resp[1]['HostedZones']:
yield 'HostedZone', _import_hosted_zone(proxy, zone)
def _import_clusters(proxy: ServiceProxy, region: str, spec: ServiceSpec):
clusters_resp = proxy.list('list_clusters')
if clusters_resp is not None:
cluster_names = clusters_resp[1].get('clusters', [])
for cluster_name in cluster_names:
yield 'Cluster', _import_cluster(proxy, cluster_name)
if resource_gate(spec, 'Nodegroup'):
yield from _import_nodegroups(proxy, cluster_name)
def _import_loadbalancers(proxy: ServiceProxy, spec: ServiceSpec):
lbs_resp = proxy.list('describe_load_balancers')
if lbs_resp is not None:
lbs = lbs_resp[1].get('LoadBalancers', [])
for lb in lbs:
yield 'LoadBalancer', _import_loadbalancer(proxy, lb)
if resource_gate(spec, 'Listener'):
yield from _import_listeners(proxy, lb['LoadBalancerArn'])
def _import_elb_region(proxy: ServiceProxy, region: str, spec: ServiceSpec):
if resource_gate(spec, 'LoadBalancer'):
_log.info(f'Importing {region} load balancers')
result = proxy.list('describe_load_balancers')
if result is not None:
elbs = result[1]
for elb in elbs.get('LoadBalancerDescriptions', []):
if not _is_v2(elb['LoadBalancerName']):
yield 'LoadBalancer', import_elb(proxy, elb)
def _import_function(proxy: ServiceProxy, function: Dict, spec: ServiceSpec):
name = function['FunctionName']
arn = function['FunctionArn']
if resource_gate(spec, 'FunctionVersion'):
versions_resp = proxy.list('list_versions_by_function',
FunctionName=name)
if versions_resp is not None:
versions = versions_resp[1]['Versions']
for version in versions:
version['ParentFunctionArn'] = arn
version['Policy'] = _get_policy(proxy, version['FunctionArn'])
yield 'FunctionVersion', version
if resource_gate(spec, 'Alias'):
aliases_resp = proxy.list('list_aliases', FunctionName=name)
if aliases_resp is not None:
aliases = aliases_resp[1]['Aliases']
for alias in aliases:
alias['FunctionArn'] = arn
alias['Policy'] = _get_policy(proxy, alias['AliasArn'])
yield 'Alias', alias
def _import_roles(proxy: ServiceProxy, spec: ServiceSpec):
if resource_gate(spec, 'role'):
roles = _unpack(proxy.list('list_roles'))
for role in roles['Roles']:
role_data = proxy.get('get_role',
RoleName=role['RoleName'])['Role']
name = role_data['RoleName']
for attr, op in _ROLE_ATTRS.items():
op_result = _unpack(proxy.list(op, RoleName=name))
role_data[attr] = op_result.get(attr)
role_data['PolicyList'] = _fetch_inline_policies(
proxy, 'role', name)
yield 'role', role_data
def _import_groups(proxy: ServiceProxy, spec: ServiceSpec):
if resource_gate(spec, 'group'):
groups = _unpack(proxy.list('list_groups'))
for group in groups.get('Groups'):
group_data = group.copy()
name = group_data['GroupName']
for attr, op in _GROUP_ATTRS.items():
op_result = proxy.list(op, GroupName=name)
if op_result is not None:
group_data[attr] = op_result[1][attr]
group_data['PolicyList'] = _fetch_inline_policies(
proxy, 'group', name)
yield 'group', group_data
def _import_ec2_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
for resource in proxy.resource_names():
_log.info(f'importing {resource}')
op_name = proxy._impl._client._PY_TO_OP_NAME[resource]
op_resource = op_name[len('Describe'):]
if op_resource in RESOURCES:
if resource_gate(spec, op_resource):
result = proxy.list(resource)
if result is not None:
if resource == 'describe_instances':
_add_user_data(proxy, result[1])
elif resource == 'describe_snapshots':
_add_launch_permissions(proxy, result[1])
elif resource == 'describe_images':
_add_image_attributes(proxy, result[1])
elif resource == 'describe_security_groups':
_add_security_group_references(proxy, result[1])
yield result[0], result[1]
_log.info(f'done with {resource}')
if resource_gate(spec, 'Defaults'):
yield from _synthesize_defaults(proxy, region)
def _import_credential_report(proxy: ServiceProxy, spec: ServiceSpec):
if resource_gate(spec, 'CredentialReport'):
# Kick off the report
started = False
init_attempts = 0
while not started:
try:
resp = proxy.get('generate_credential_report')
started = resp.get('State') in ('STARTED', 'COMPLETE')
except ClientError as e:
code = e.response.get('Error', {}).get('Code')
is_throttled = code == 'Throttling'
if not is_throttled:
_log.error('credential report error', exc_info=e)
raise
# wait and try again?
init_attempts += 1
if init_attempts >= 3:
_log.error('credential report error', exc_info=e)
raise GFError('Failed to generate credential report')
time.sleep(0.1)
except SSLError:
# wait and try again?
init_attempts += 1
if init_attempts >= 3:
raise GFError(
'Failed to generate credential report, SSL Error')
time.sleep(0.1)
attempts = 0
report = None
while attempts < 20 and report is None:
try:
report = proxy.get('get_credential_report')
except ClientError as e:
code = e.response.get('Error', {}).get('Code')
if code == 'ReportInProgress':
attempts += 1
time.sleep(1)
else:
_log.error('credenetial report fetch error', exc_info=e)
raise
if report is None:
raise GFError('Failed to fetch credential report')
decoded = report['Content'].decode('utf-8')
reader = csv.DictReader(StringIO(decoded))
for row in reader:
processed = _post_process_report_row(row)
yield 'CredentialReport', processed
def _import_policies(proxy: ServiceProxy, spec: ServiceSpec):
if resource_gate(spec, 'policy'):
results = proxy.list('list_policies', Scope='Local')
if results is not None:
policies = results[1]
for policy in policies['Policies']:
policy_data = policy.copy()
arn = policy_data['Arn']
versions_result = _unpack(
proxy.list('list_policy_versions', PolicyArn=arn))
versions = []
for version in versions_result.get('Versions', []):
full_version = proxy.get('get_policy_version',
PolicyArn=arn,
VersionId=version['VersionId'])
versions.append(full_version['PolicyVersion'])
policy_data['Versions'] = versions
op_result = proxy.list('list_entities_for_policy',
PolicyArn=arn)
if op_result is not None:
for attr in ['PolicyGroups', 'PolicyUsers', 'PolicyRoles']:
policy_data[attr] = op_result[1][attr]
yield 'policy', policy_data
# TODO: fix this cut + paste
aws_policies = proxy.list('list_policies',
Scope='AWS',
OnlyAttached=True)
if aws_policies is not None:
policies = aws_policies[1]
for policy in policies['Policies']:
policy_data = policy.copy()
arn = policy_data['Arn']
versions_result = _unpack(
proxy.list('list_policy_versions', PolicyArn=arn))
versions = []
for version in versions_result.get('Versions', []):
full_version = proxy.get('get_policy_version',
PolicyArn=arn,
VersionId=version['VersionId'])
versions.append(full_version['PolicyVersion'])
policy_data['Versions'] = versions
op_result = proxy.list('list_entities_for_policy',
PolicyArn=arn)
if op_result is not None:
for attr in ['PolicyGroups', 'PolicyUsers', 'PolicyRoles']:
policy_data[attr] = op_result[1][attr]
yield 'policy', policy_data
def _import_users(proxy: ServiceProxy, spec: ServiceSpec):
if resource_gate(spec, 'user'):
users = _unpack(proxy.list('list_users'))
for user in users['Users']:
user_data = user.copy()
name = user_data['UserName']
for attr, op_desc in _USER_ATTRS.items():
if isinstance(op_desc, str):
op = op_desc
field = attr
else:
op = op_desc['op']
field = op_desc['key']
op_result = proxy.list(op, UserName=name)
if op_result is not None:
user_data[attr] = op_result[1][field]
user_data['PolicyList'] = _fetch_inline_policies(
proxy, 'user', name)
login_profile = proxy.get('get_login_profile', UserName=name)
if login_profile is not None:
login_profile = login_profile['LoginProfile']
user_data['LoginProfile'] = login_profile
yield 'user', user_data
def _import_logs_region(proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
if resource_gate(spec, 'LogGroup'):
_log.info(f'Import LogGroups in {region}')
yield from _import_log_groups(proxy)
def _import_cloudfront(proxy: ServiceProxy,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'importing cloudfront distributions')
if resource_gate(spec, 'Distribution'):
yield from _import_distributions(proxy)
def _import_efs_region(proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'import efs in {region}')
if resource_gate(spec, 'FileSystem'):
yield from _import_filesystems(proxy)
def _import_glacier_region(proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'import glacier in {region}')
if resource_gate(spec, 'Vault'):
yield from _import_vaults(proxy)
def _import_acm_pca_region(proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'import acm-pca in {region}')
if resource_gate(spec, 'CertificateAuthority'):
yield from _import_certificate_authorities(proxy)
def _import_sqs_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
if resource_gate(spec, 'Queue'):
_log.info(f'importing sqs Queues in {region}')
yield from _import_queues(proxy, region)
def _import_kms_region(proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'import kms in {region}')
if resource_gate(spec, 'Key'):
yield from _import_keys(proxy)
def _import_cloudformation_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Iterator[Tuple[str, Any]]:
_log.info(f'import cloudformation in {region}')
if resource_gate(spec, 'Stack'):
yield from _import_cloudformation_stacks(proxy)
def _import_ssm_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
if resource_gate(spec, 'Parameter'):
_log.info(f'importing describe_parameters {region}')
yield from _import_parameters(proxy, region)
def _import_cloudtrail_region(
proxy: ServiceProxy, region: str,
spec: ServiceSpec) -> Generator[Tuple[str, Any], None, None]:
_log.info(f'importing describe_trails in {region}')
if resource_gate(spec, 'Trail'):
yield from _import_trails(proxy, region)
