def disable(config, tags, accounts, master, debug, suspend, disable_detector, delete_detector, dissociate): """suspend guard duty in the given accounts.""" accounts_config, master_info, executor = guardian_init( config, debug, master, accounts, tags) if sum(map(int, (suspend, disable_detector, dissociate))) != 1: raise ValueError( ("One and only of suspend, disable-detector, dissociate" "can be specified.")) master_session = assumed_session(master_info['role'], 'c7n-guardian') master_client = master_session.client('guardduty') detector_id = get_or_create_detector_id(master_client) if suspend: unprocessed = master_client.stop_monitoring_members( DetectorId=detector_id, AccountIds=[a['account_id'] for a in accounts_config['accounts'] ]).get('UnprocessedAccounts', ()) if unprocessed: log.warning("Following accounts where unprocessed\n %s", format_event(unprocessed)) log.info("Stopped monitoring %d accounts in master", len(accounts_config['accounts'])) return if dissociate: master_client.disassociate_members( DetectorId=detector_id, AccountIds=[a['account_id'] for a in accounts_config['accounts']]) # Seems like there's a couple of ways to disable an account # delete the detector (member), disable the detector (master or member), # or disassociate members, or from member disassociate from master. for a in accounts_config['accounts']: member_session = assumed_session(master_info['role'], 'c7n-guardian') member_client = member_session.client('guardduty') m_detector_id = get_or_create_detector_id(member_client) if disable_detector: member_client.update_detector(DetectorId=m_detector_id, Enable=False) log.info("Disabled detector in account:%s", a['name']) if dissociate: try: log.info("Disassociated member account:%s", a['name']) result = member_client.disassociate_from_master_account( DetectorId=m_detector_id) log.info("Result %s", format_event(result)) except ClientError as e: if e.response['Error']['Code'] == 'InvalidInputException': continue if delete_detector: member_client.delete_detector(DetectorId=m_detector_id) log.info("Deleted detector in account:%s", a['name'])
def get_session(role, session_name="c7n-log-exporter", session=None): if role == 'self': session = boto3.Session() elif isinstance(role, basestring): session = assumed_session(role, session_name) elif isinstance(role, list): session = None for r in role: session = assumed_session(r, session_name, session=session) else: session = boto3.Session() return session
def get_session(role, region, session_name="c7n-log-exporter", session=None): if role == 'self': session = boto3.Session() elif isinstance(role, six.string_types): session = assumed_session(role, session_name, region=region) elif isinstance(role, list): session = None for r in role: session = assumed_session(r, session_name, session=session, region=region) else: session = boto3.Session() return session
def test_assumed_session(self): factory = self.replay_flight_data("test_credential_sts") session = assumed_session( role_arn='arn:aws:iam::644160558196:role/CustodianGuardDuty', session_name="custodian-dev", session=factory(), ) # attach the placebo flight recorder to the new session. pill = placebo.attach( session, os.path.join(self.placebo_dir, 'test_credential_sts')) if self.recording: pill.record() else: pill.playback() self.addCleanup(pill.stop) try: identity = session.client("sts").get_caller_identity() except ClientError as e: self.assertEqual(e.response["Error"]["Code"], "ValidationError") self.assertEqual( identity['Arn'], 'arn:aws:sts::644160558196:assumed-role/CustodianGuardDuty/custodian-dev' )
def run_account_script(account, region, output_dir, debug, script_args): try: session = assumed_session(account['role'], "org-script", region=region) creds = session._session.get_credentials() except: log.error( "unable to obtain credentials for account:%s role:%s", account['name'], account['role']) return 1 env = os.environ.copy() env['AWS_ACCESS_KEY_ID'] = creds.access_key env['AWS_SECRET_ACCESS_KEY'] = creds.secret_key env['AWS_SESSION_TOKEN'] = creds.token env['AWS_DEFAULT_REGION'] = region log.info("running script on account:%s region:%s script: `%s`", account['name'], region, " ".join(script_args)) if debug: subprocess.check_call(args=script_args, env=env) return 0 output_dir = os.path.join(output_dir, account['name'], region) if not os.path.exists(output_dir): os.makedirs(output_dir) with open(os.path.join(output_dir, 'stdout'), 'wb') as stdout: with open(os.path.join(output_dir, 'stderr'), 'wb') as stderr: return subprocess.call( args=script_args, env=env, stdout=stdout, stderr=stderr)
def get_session(account, session_name, region): if account.get('provider') != 'aws': return None if account.get('role'): roles = account['role'] if isinstance(roles, str): roles = [roles] s = None for r in roles: try: s = assumed_session(r, session_name, region=region, external_id=account.get('external_id'), session=s) except ClientError as e: log.error( "unable to obtain credentials for account:%s role:%s error:%s", account['name'], r, e) raise return s elif account.get('profile'): return SessionFactory(region, account['profile'])() else: raise ValueError("No profile or role assume specified for account %s" % account)
def report(config, tags, accounts, master, debug): """report on guard duty enablement by account""" accounts_config, master_info, executor = guardian_init( config, debug, master, accounts, tags) session = assumed_session(master_info['role'], 'c7n-guardian') client = session.client('guardduty') detector_id = get_or_create_detector_id(client) members = { m['AccountId']: m for m in client.list_members(DetectorId=detector_id).get('Members') } accounts_report = [] for a in accounts_config['accounts']: ar = dict(a) accounts_report.append(ar) ar.pop('tags', None) ar.pop('role') ar.pop('regions', None) if a['account_id'] not in members: ar['member'] = False ar['status'] = None ar['invited'] = None ar['updated'] = None continue m = members[a['account_id']] ar['status'] = m['RelationshipStatus'] ar['member'] = True ar['joined'] = m['InvitedAt'] ar['updated'] = m['UpdatedAt'] accounts_report.sort(key=operator.itemgetter('updated'), reverse=True) print(tabulate(accounts_report, headers=('keys')))
def run_account_script(account, region, output_dir, debug, script_args): try: session = assumed_session(account['role'], "org-script", region=region) creds = session._session.get_credentials() except: log.error("unable to obtain credentials for account:%s role:%s", account['name'], account['role']) return 1 env = os.environ.copy() env['AWS_ACCESS_KEY_ID'] = creds.access_key env['AWS_SECRET_ACCESS_KEY'] = creds.secret_key env['AWS_SESSION_TOKEN'] = creds.token env['AWS_DEFAULT_REGION'] = region log.info("running script on account:%s region:%s script: `%s`", account['name'], region, " ".join(script_args)) if debug: subprocess.check_call(args=script_args, env=env) return 0 output_dir = os.path.join(output_dir, account['name'], region) if not os.path.exists(output_dir): os.makedirs(output_dir) with open(os.path.join(output_dir, 'stdout'), 'wb') as stdout: with open(os.path.join(output_dir, 'stderr'), 'wb') as stderr: return subprocess.call(args=script_args, env=env, stdout=stdout, stderr=stderr)
def get_sessions(accounts_config, account_ids): sessions = {} for a in accounts_config.get('accounts', []): if a['account_id'] not in account_ids: continue session = assumed_session(a['role'], 'app-metrics') sessions[a['account_id']] = session return sessions
def index_account_trails(config, account, region, date, directory): es_client = get_es_client(config) s3 = local_session( lambda: assumed_session(account['role'], 'TrailIndex')).client('s3') bucket = account['bucket'] key_prefix = "accounts/{}/{}/traildb".format(account['name'], region) marker = + "{}/{}/trail.db.bz2".format(key_prefix, date) p = s3.get_paginator('list_objects_v2').paginate( Bucket=bucket, Prefix=key_prefix, StartAfter=marker, ) with ThreadPoolExecutor(max_workers=20) as w: for key_set in p: if 'Contents' not in key_set: continue keys = [] for k in key_set['Contents']: if (k['Key'].endswith('trail.db.bz2') and valid_date(k['Key'], date)): keys.append(k) futures = map( lambda k: w.submit( get_traildb, bucket, k, lambda: assumed_session( account['role'], 'TrailIndex'), directory), keys) for f in as_completed(futures): local_db_file = f.result() connection = sqlite3.connect(local_db_file) connection.row_factory = dict_factory cursor = connection.cursor() index_events(es_client, fetch_events(cursor, config, account['name'])) connection.close() try: os.remove(local_db_file) except: log.warning("Failed to remove temporary file: {}".format( local_db_file)) pass
def get_session(account, session_name, region): if account.get('role'): return assumed_session(role, session_name, region=region) elif account.get('profile') return SessionFactory(region, profile)() else: raise ValueError( "No profile or role assume specified for account %s" % account)
def get_session(account, session_name, region): if account.get('role'): return assumed_session(account['role'], session_name, region=region, external_id=account.get('external_id')) elif account.get('profile'): return SessionFactory(region, account['profile'])() else: raise ValueError( "No profile or role assume specified for account %s" % account)
def get_session(role, session_name="c7n-log-exporter"): if role == 'self': session = boto3.Session() elif role: session = assumed_session(role, session_name) else: session = boto3.Session() return session
def get_session(role, session_name, profile): region = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') stats = ApiStats(Bag(), Config.empty()) if role: s = assumed_session(role, session_name, region=region) else: s = SessionFactory(region, profile)() stats(s) return stats, s
def index_account_metrics(config, idx_name, region, account, start, end, period): session = assumed_session(account['role'], 'PolicyIndex') indexer = get_indexer(config) client = session.client('cloudwatch', region_name=region) policies = set() account_metrics = [] pager = client.get_paginator('list_metrics') for p in pager.paginate(Namespace=NAMESPACE): metrics = p.get('Metrics') for metric in metrics: if 'Dimensions' not in metric: log.warning("account:%s region:%s metric with no dims: %s", account['name'], region, metric) continue dims = { d['Name']: d['Value'] for d in metric.get('Dimensions', ()) } if dims['Policy'] not in policies: log.debug("Processing account:%s region:%s policy: %s", account['name'], region, dims['Policy']) policies.add(dims['Policy']) account_metrics.append(metric) for p in pager.paginate(Namespace='AWS/Lambda'): metrics = p.get('Metrics') for metric in metrics: dims = { d['Name']: d['Value'] for d in metric.get('Dimensions', ()) } if not dims.get('FunctionName', '').startswith('custodian-'): continue account_metrics.append(metric) log.debug("account:%s region:%s processing metrics:%d start:%s end:%s", account['name'], region, len(account_metrics), start.strftime("%Y/%m/%d"), end.strftime("%Y/%m/%d")) region_time = region_points = 0 # originally was parallel thread, but rate limits around get # metric stat polling means single threaded is faster. for metric_set in chunks(account_metrics, 20): mt, mp = index_metric_set(indexer, account, region, metric_set, start, end, period) region_time += mt region_points += mp log.info(("indexed account:%s region:%s metrics:%d" " points:%d start:%s end:%s time:%0.2f"), account['name'], region, len(account_metrics), region_points, start.strftime("%Y/%m/%d"), end.strftime("%Y/%m/%d"), region_time) return region_time, region_points
def get_session(account, session_name, region): if account.get('role'): return assumed_session( account['role'], session_name, region=region, external_id=account.get('external_id')) elif account.get('profile'): return SessionFactory(region, account['profile'])() else: raise ValueError( "No profile or role assume specified for account %s" % account)
def get_clients(accounts_config, account_ids, regions, service='cloudwatch'): clients = {} for a in accounts_config.get('accounts', []): if a['account_id'] not in account_ids: continue session = assumed_session(a['role'], 'app-metrics') for r in regions: clients['%s-%s' % (a['account_id'], r)] = session.client( service, region_name=r) return clients
def get_clients(accounts_config, account_ids, regions, service='cloudwatch'): clients = {} for a in accounts_config.get('accounts', []): if a['account_id'] not in account_ids: continue session = assumed_session(a['role'], 'app-metrics') for r in regions: clients['%s-%s' % ( a['account_id'], r)] = session.client(service, region_name=r) return clients
def get_session(self, account_id): """Get an active session in the target account.""" if account_id not in self.account_sessions: if account_id not in self.config['accounts']: raise AccountNotFound("account:%s is unknown" % account_id) self.account_sessions[account_id] = s = assumed_session( self.config['accounts'][account_id]['role'], "Sphere11") s._session.user_agent_name = "Sphere11" s._session.user_agent_version = "0.07" return self.account_sessions[account_id]
def get_api_credentials(self): session = local_session(self.manager.session_factory) if self.data.get('role'): api_session = assumed_session(self.data.get('role'), 'CustodianSphere11', session) else: api_session = session credentials = api_session.get_credentials() region = self.data.get('region', 'us-east-1') auth = SignatureAuth(credentials, region, 'execute-api') return auth
def get_api_credentials(self): session = local_session(self.manager.session_factory) if self.data.get('role'): api_session = assumed_session( self.data.get('role'), 'CustodianSphere11', session) else: api_session = session credentials = api_session.get_credentials() region = self.data.get('region', 'us-east-1') auth = SignatureAuth(credentials, region, 'execute-api') return auth
def get_session(account_info): s = getattr(CONN_CACHE, '%s-session', None) t = getattr(CONN_CACHE, 'time', 0) n = time.time() if s is not None and t + (60 * random.uniform(20, 45)) > n: return s if account_info.get('role'): s = assumed_session(account_info['role'], SESSION_NAME) else: s = boto3.Session() CONN_CACHE.session = s CONN_CACHE.time = n return s
def index_account_metrics(config, idx_name, region, account, start, end, period): session = assumed_session(account['role'], 'PolicyIndex') indexer = get_indexer(config) client = session.client('cloudwatch', region_name=region) policies = set() account_metrics = [] pager = client.get_paginator('list_metrics') for p in pager.paginate(Namespace=NAMESPACE): metrics = p.get('Metrics') for metric in metrics: if 'Dimensions' not in metric: log.warning("account:%s region:%s metric with no dims: %s", account['name'], region, metric) continue dims = {d['Name']: d['Value'] for d in metric.get( 'Dimensions', ())} if dims['Policy'] not in policies: log.debug("Processing account:%s region:%s policy: %s", account['name'], region, dims['Policy']) policies.add(dims['Policy']) account_metrics.append(metric) for p in pager.paginate(Namespace='AWS/Lambda'): metrics = p.get('Metrics') for metric in metrics: dims = {d['Name']: d['Value'] for d in metric.get('Dimensions', ())} if not dims.get('FunctionName', '').startswith('custodian-'): continue account_metrics.append(metric) log.debug("account:%s region:%s processing metrics:%d start:%s end:%s", account['name'], region, len(account_metrics), start.strftime("%Y/%m/%d"), end.strftime("%Y/%m/%d")) region_time = region_points = 0 # originally was parallel thread, but rate limits around get # metric stat polling means single threaded is faster. for metric_set in chunks(account_metrics, 20): mt, mp = index_metric_set( indexer, account, region, metric_set, start, end, period) region_time += mt region_points += mp log.info(("indexed account:%s region:%s metrics:%d" " points:%d start:%s end:%s time:%0.2f"), account['name'], region, len(account_metrics), region_points, start.strftime("%Y/%m/%d"), end.strftime("%Y/%m/%d"), region_time) return region_time, region_points
def get_session(account_info): """Get a boto3 sesssion potentially cross account sts assumed assumed sessions are automatically refreshed. """ s = getattr(CONN_CACHE, '%s-session' % account_info['name'], None) if s is not None: return s if account_info.get('role'): s = assumed_session(account_info['role'], SESSION_NAME) else: s = boto3.Session() setattr(CONN_CACHE, '%s-session' % account_info['name'], s) return s
def xtest_assumed_session(self): # placebo's datetime bug bites again # https://github.com/garnaat/placebo/pull/50 factory = self.replay_flight_data('test_credential_sts') user = factory().client('iam').get_user() session = assumed_session( "arn:aws:iam::644160558196:role/CloudCustodianRole", "custodian-dev", session=factory()) try: session.client('iam').get_user() except ClientError as e: self.assertEqual(e.response['Error']['Code'], 'ValidationError') else: self.fail("sts user not identifyable this way") self.assertEqual(user['User']['UserName'], 'kapil')
def index_account_resources(config, account, region, policy, date): indexer = get_indexer(config, type=policy['resource']) bucket = account['bucket'] key_prefix = "accounts/{}/{}/policies/{}".format(account['name'], region, policy['name']) records = s3_resource_parser.record_set( lambda: assumed_session(account['role'], 'PolicyIndex'), bucket, key_prefix, date, specify_hour=True) for r in records: r['c7n:MatchedPolicy'] = policy['name'] indexer.index(records)
def enable_account(account, master_account_id, region): member_session = assumed_session(account['role'], 'c7n-guardian', region=region) member_client = member_session.client('guardduty') m_detector_id = get_or_create_detector_id(member_client) invitations = [ i for i in member_client.list_invitations().get('Invitations', []) if i['AccountId'] == master_account_id ] invitations.sort(key=operator.itemgetter('InvitedAt')) if not invitations: log.warning("No guard duty invitation found for %s id:%s" ( account['name'])) return member_client.accept_invitation( DetectorId=m_detector_id, InvitationId=invitations[-1]['InvitationId'], MasterId=master_account_id) return True
def index_metric_set(indexer, account, region, metric_set, start, end, period): session = local_session( lambda : assumed_session(account['role'], 'PolicyIndex')) # NOQA E203 client = session.client('cloudwatch', region_name=region) t = time.time() account_info = dict(account['tags']) account_info['Account'] = account['name'] account_info['AccountId'] = account['id'] account_info['Region'] = region point_count = 0 for m in metric_set: params = dict( Namespace=m['Namespace'], MetricName=m['MetricName'], Statistics=['Sum'], Dimensions=m['Dimensions'], StartTime=start, EndTime=end, Period=period) try: points = retry(client.get_metric_statistics, **params)['Datapoints'] except Exception as e: log.error( "error account:%s region:%s start:%s end:%s error:%s", account['name'], region, start, end, e) if not points: continue dims = {d['Name']: d['Value'] for d in m.pop('Dimensions', ())} for p in points: if m['Namespace'] == 'AWS/Lambda': dims['Policy'] = dims['FunctionName'].split('-', 1)[1] p.update(dims) p.update(m) p.update(account_info) point_count += len(points) log.debug("account:%s region:%s metric:%s points:%d policy:%s", account['name'], region, m['MetricName'], len(points), dims.get('Policy', 'unknown')) indexer.index(points) return time.time() - t, point_count
def get_session(account, session_name, region): if account.get('role'): roles = account['role'] if isinstance(roles, six.string_types): roles = [roles] s = None for r in roles: try: s = assumed_session( r, session_name, region=region, external_id=account.get('external_id'), session=s) except ClientError as e: log.error( "unable to obtain credentials for account:%s role:%s error:%s", account['name'], r, e) raise return s elif account.get('profile'): return SessionFactory(region, account['profile'])() else: raise ValueError( "No profile or role assume specified for account %s" % account)
def get_session(role, session_name, profile): region = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') if role: return assumed_session(role, session_name, region=region) else: return SessionFactory(region, profile)()
def enable(config, master, tags, accounts, debug, message, region): """enable guard duty on a set of accounts""" accounts_config, master_info, executor = guardian_init( config, debug, master, accounts, tags) master_session = assumed_session(master_info['role'], 'c7n-guardian', region=region) master_client = master_session.client('guardduty') detector_id = get_or_create_detector_id(master_client) extant_members = master_client.list_members(DetectorId=detector_id).get( 'Members', ()) extant_ids = {m['AccountId'] for m in extant_members} # Find extant members not currently enabled suspended_ids = { m['AccountId'] for m in extant_members if m['RelationshipStatus'] == 'Disabled' } # Filter by accounts under consideration per config and cli flags suspended_ids = { a['account_id'] for a in accounts_config['accounts'] if a['account_id'] in suspended_ids } if suspended_ids: unprocessed = master_client.start_monitoring_members( DetectorId=detector_id, AccountIds=list(suspended_ids)).get('UnprocessedAccounts') if unprocessed: log.warning("Unprocessed accounts on re-start monitoring %s" % (format_event(unprocessed))) log.info("Restarted monitoring on %d accounts" % (len(suspended_ids))) members = [{ 'AccountId': account['account_id'], 'Email': account['email'] } for account in accounts_config['accounts'] if account['account_id'] not in extant_ids] if not members: if not suspended_ids: log.info("All accounts already enabled") return if (len(members) + len(extant_ids)) > 100: raise ValueError( "Guard Duty only supports 100 member accounts per master account") log.info("Enrolling %d accounts in guard duty" % len(members)) log.info("Creating member accounts") unprocessed = master_client.create_members( DetectorId=detector_id, AccountDetails=members).get('UnprocessedAccounts') if unprocessed: log.warning("Following accounts where unprocessed\n %s" % format_event(unprocessed)) log.info("Inviting member accounts") params = { 'AccountIds': [m['AccountId'] for m in members], 'DetectorId': detector_id } if message: params['Message'] = message unprocessed = master_client.invite_members( **params).get('UnprocessedAccounts') if unprocessed: log.warning("Following accounts where unprocessed\n %s" % format_event(unprocessed)) log.info("Accepting invitations") with executor(max_workers=WORKER_COUNT) as w: futures = {} for a in accounts_config['accounts']: if a == master_info: continue futures[w.submit(enable_account, a, master_info['account_id'], region)] = a for f in as_completed(futures): a = futures[f] if f.exception(): log.error("Error processing account:%s error:%s", f.exception()) continue if f.result(): log.info('Enabled guard duty on account:%s' % account['name'])
def get_session(role, session_name, profile, region): if role: return assumed_session(role, session_name, region=region) else: return SessionFactory(region, profile)()