def build_aws_import_job(db: Session, session: boto.Session,
confirm: ConfirmAcct,
external_id: Optional[int]) -> ImportJob:
proxy = Proxy.build(session)
sts = session.create_client('sts')
identity = sts.get_caller_identity()
provider = _get_or_create_provider(db, proxy, identity, confirm,
external_id)
desc = _build_import_job_desc(proxy, identity)
org_id = desc['aws_org']['Id']
return ImportJob.create(provider, desc, org_id)
def import_to_db(db: Session, import_job_id: int,
service_spec: ServiceSpec):
job = db.query(ImportJob).get(import_job_id)
if job is None:
raise GFInternal('Lost ImportJob')
writer = db_import_writer(db,
job.id,
job.provider_account_id,
self.name,
phase=0,
source='base')
for path, account in account_paths_for_import(db, job):
boto = load_boto_session(account)
proxy = Proxy.build(boto)
ps = PathStack.from_import_job(job)
service_proxy = proxy.service(self.name)
for fn in resource_fns:
fn(service_proxy, writer, account.scope, ps, service_spec)
def _global_async_proxy(ps: PathStack, import_job_id: int,
provider_account_id: int, config: Dict, svc_name: str,
account_id: str, service_spec: ServiceSpec,
import_fn: GlobalResourceSpec):
db = import_session()
boto = load_boto_session_from_config(config)
proxy = Proxy.build(boto)
service_proxy = proxy.service(svc_name)
writer = db_import_writer(db,
import_job_id,
provider_account_id,
svc_name,
phase=0,
source='base')
try:
import_fn(service_proxy, writer, account_id, ps, service_spec)
except Exception as e:
_log.error(f'Failed for svc {svc_name}', exc_info=e)
raise
db.commit()
def _async_proxy(ps: PathStack, import_job_id: int, provider_account_id: int,
region: str, config: Dict, svc_name: str,
service_spec: ServiceSpec, import_fn: RegionalImportFn):
db = import_session()
ps = ps.scope(region)
boto = load_boto_session_from_config(config)
proxy = Proxy.build(boto)
service_proxy = proxy.service(svc_name, region)
writer = db_import_writer(db,
import_job_id,
provider_account_id,
svc_name,
phase=0,
source='base')
_log.debug(f'Starting {svc_name} - {region}')
for resource_name, raw_resources in import_fn(service_proxy, region,
service_spec):
writer(ps, resource_name, raw_resources, {'region': region})
db.commit()
_log.debug(f'Committed {svc_name} - {region}')
def import_region_to_db(db: Session, import_job_id: int, region: str,
service_spec: ServiceSpec):
job = db.query(ImportJob).get(import_job_id)
if job is None:
raise GFInternal('Lost ImportJob')
writer = db_import_writer(db,
job.id,
job.provider_account_id,
svc_name,
phase=0,
source='base')
for path, account in account_paths_for_import(db, job):
boto = load_boto_session(account)
proxy = Proxy.build(boto)
ps = PathStack.from_import_job(job).scope(account.scope)
service_proxy = proxy.service(svc_name, region)
ps = ps.scope(region)
for resource_name, raw_resources in fn(service_proxy, region,
service_spec):
writer(ps, resource_name, raw_resources, {'region': region})
def map_import(db: Session, import_job_id: int, partition: str,
spec: ImportSpec):
import_job = db.query(ImportJob).get(import_job_id)
if import_job is None:
raise GFInternal('Lost ImportJob')
ps = PathStack.from_import_job(import_job)
mapper = _get_mapper(import_job)
gate = service_gate(spec)
for path, account in account_paths_for_import(db, import_job):
uri_fn = get_arn_fn(account.scope, partition)
ps = PathStack.from_import_job(import_job).scope(account.scope)
map_resource_prefix(db, import_job, ps.path(), mapper, uri_fn)
boto = None
proxy = None
if gate('iam') is not None:
boto = load_boto_session(account)
proxy = Proxy.build(boto)
synthesize_account_root(proxy, db, import_job, ps.path(),
account.scope, partition)
ec2_spec = gate('ec2')
if ec2_spec is not None and resource_gate(ec2_spec, 'Images'):
# Additional ec2 work
if boto is None or proxy is None:
boto = load_boto_session(account)
proxy = Proxy.build(boto)
adjunct_writer = db_import_writer(db,
import_job.id,
import_job.provider_account_id,
'ec2',
phase=1,
source='base')
find_adjunct_data(db, proxy, adjunct_writer, import_job, ps,
import_job)
logs_spec = gate('logs')
if logs_spec is not None and resource_gate(logs_spec,
'ResourcePolicies'):
if boto is None or proxy is None:
boto = load_boto_session(account)
proxy = Proxy.build(boto)
region_cache = RegionCache(boto, partition)
adjunct_writer = db_import_writer(db,
import_job.id,
import_job.provider_account_id,
'logs',
phase=1,
source='logspolicies')
add_logs_resource_policies(db, proxy, region_cache, adjunct_writer,
import_job, ps, account.scope)
for source in AWS_SOURCES:
map_partial_prefix(db, mapper, import_job, source, ps.path(),
uri_fn)
map_partial_deletes(db, import_job, ps.path(), source, spec)
# Re-map anything we've added
map_resource_prefix(db, import_job, ps.path(), mapper, uri_fn)
# Handle deletes
map_resource_deletes(db, ps.path(), import_job, spec)
found_relations = map_resource_relations(db, import_job, ps.path(),
mapper, uri_fn)
map_relation_deletes(db, import_job, ps.path(), found_relations, spec)