def _load_source_credentials(cls, account, region_name): key_id = account.access_key_id secret_key = account.secret_access_key token = None expiration = None if account.is_ec2_instance_role: logger.info('fetch ec2 instance credentials') fetcher = InstanceMetadataFetcher(timeout=5.0, num_attempts=10) response = fetcher.retrieve_iam_role_credentials() key_id = response['access_key'] secret_key = response['secret_key'] token = response['token'] expiration = response['expiry_time'] if not isinstance(expiration, datetime): expiration = parse_datetime(expiration) sts = cls.create_sts_client(key_id, secret_key, token, region_name) identity = sts.get_caller_identity() arn = identity.get('Arn') credentials = AWSCredentials() credentials.aws_access_key_id = key_id credentials.aws_secret_access_key = secret_key credentials.aws_session_token = token credentials.expiration = expiration credentials.category = account.category credentials.arn = arn return credentials
def get_instance_metadata(self, metadata_type): """This method fetches instance metadata using AWS metadata api Args: metadata_type (str): Metadata to fetch Returns: metadata (str): metadata information for the requested metadata key or raises a runtime exception. """ if metadata_type in ["mac", "instance-id", "security-groups"]: return urlopen( os.path.join(self.INSTANCE_METADATA_API, metadata_type), timeout=self.METADATA_API_TIMEOUT_SECONDS).read().decode( 'utf-8') elif metadata_type in ["vpc-id", "subnet-id"]: mac = self.get_instance_metadata("mac") return urlopen( os.path.join(self.NETWORK_METADATA_API, mac, metadata_type), timeout=self.METADATA_API_TIMEOUT_SECONDS).read().decode( 'utf-8') elif metadata_type in ["region", "privateIp"]: identity_data = urlopen(self.INSTANCE_IDENTITY_API, timeout=self.METADATA_API_TIMEOUT_SECONDS) \ .read().decode('utf-8') return json.loads(identity_data).get( metadata_type) if identity_data else None elif metadata_type in ["role"]: # Arg timeout is in MS. fetcher = InstanceMetadataFetcher( timeout=self.METADATA_API_TIMEOUT_SECONDS, num_attempts=2) c = fetcher.retrieve_iam_role_credentials() # This will return None in case of no assigned role on the instance. return c.get("role_name") else: raise YBOpsRuntimeError( "Unsupported metadata type: {}".format(metadata_type))
def test_catch_invalid_imds_error(self): with mock.patch('botocore.httpsession.URLLib3Session.send') as send_mock: fetcher = InstanceMetadataFetcher() e = LocationParseError(location="foo") send_mock.side_effect = HTTPClientError(error=e) with self.assertRaises(InvalidIMDSEndpointError): fetcher.retrieve_iam_role_credentials()
def __init__(self, retries=DEFAULT_RETRIES, metadata_service_timeout=DEFAULT_METADATA_SERVICE_TIMEOUT): self.instance_metadata_fetcher = InstanceMetadataFetcher() self.instance_metadata_fetcher._num_attempts = retries self.instance_metadata_fetcher._timeout = metadata_service_timeout self.instance_metadata_fetcher._needs_retry_for_credentials = self.needs_retry_for_credentials self.instance_metadata_provider = InstanceMetadataProvider( self.instance_metadata_fetcher)
def test_catch_retryable_http_errors(self): with mock.patch('botocore.httpsession.URLLib3Session.send') as send_mock: fetcher = InstanceMetadataFetcher() send_mock.side_effect = ConnectionClosedError(endpoint_url="foo") creds = fetcher.retrieve_iam_role_credentials() self.assertEqual(send_mock.call_count, 2) for call_instance in send_mock.call_args_list: self.assertTrue(call_instance[0][0].url.startswith(fetcher.get_base_url())) self.assertEqual(creds, {})
def load(self): timeout = self.session.get_config_variable('metadata_service_timeout') num_attempts = self.session.get_config_variable( 'metadata_service_num_attempts') retrieve_kwargs = {} if timeout is not None: retrieve_kwargs['timeout'] = float(timeout) if num_attempts is not None: retrieve_kwargs['num_attempts'] = int(num_attempts) fetcher = InstanceMetadataFetcher() # Partially apply the arguments for future calls. refresh_using = functools.partial( fetcher.retrieve_iam_role_credentials, **retrieve_kwargs) # We do the first request, to see if we get useful data back. # If not, we'll pass & move on to whatever's next in the credential # chain. metadata = refresh_using() if not metadata: return None logger.info('Found IAM Role: %s', metadata['role_name']) # We manually set the data here, since we already made the request & # have it. When the expiry is hit, the credentials will auto-refresh # themselves. creds = RefreshableCredentials.create_from_metadata( metadata, method=self.method, refresh_using=refresh_using) return creds
def create_credential_resolver(session): """Create a default credential resolver. This creates a pre-configured credential resolver that includes the default lookup chain for credentials. """ profile_name = session.get_config_variable('profile') or 'default' credential_file = session.get_config_variable('credentials_file') config_file = session.get_config_variable('config_file') metadata_timeout = session.get_config_variable('metadata_service_timeout') num_attempts = session.get_config_variable('metadata_service_num_attempts') env_provider = EnvProvider() providers = [ env_provider, AssumeRoleProvider( load_config=lambda: session.full_config, client_creator=session.create_client, cache={}, profile_name=profile_name, ), SharedCredentialProvider(creds_filename=credential_file, profile_name=profile_name), # The new config file has precedence over the legacy # config file. ConfigProvider(config_filename=config_file, profile_name=profile_name), OriginalEC2Provider(), BotoProvider(), ContainerProvider(), InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher( timeout=metadata_timeout, num_attempts=num_attempts)) ] explicit_profile = session.get_config_variable('profile', methods=('instance', )) if explicit_profile is not None: # An explicitly provided profile will negate an EnvProvider. # We will defer to providers that understand the "profile" # concept to retrieve credentials. # The one edge case if is all three values are provided via # env vars: # export AWS_ACCESS_KEY_ID=foo # export AWS_SECRET_ACCESS_KEY=bar # export AWS_PROFILE=baz # Then, just like our client() calls, the explicit credentials # will take precedence. # # This precedence is enforced by leaving the EnvProvider in the chain. # This means that the only way a "profile" would win is if the # EnvProvider does not return credentials, which is what we want # in this scenario. providers.remove(env_provider) else: logger.debug('Skipping environment variable credential check' ' because profile name was explicitly set.') resolver = CredentialResolver(providers=providers) return resolver
def create_credential_resolver(session): """Create a default credential resolver. This creates a pre-configured credential resolver that includes the default lookup chain for credentials. """ profile_name = session.get_config_variable('profile') or 'default' credential_file = session.get_config_variable('credentials_file') config_file = session.get_config_variable('config_file') metadata_timeout = session.get_config_variable('metadata_service_timeout') num_attempts = session.get_config_variable('metadata_service_num_attempts') providers = [ EnvProvider(), SharedCredentialProvider(creds_filename=credential_file, profile_name=profile_name), # The new config file has precedence over the legacy # config file. ConfigProvider(config_filename=config_file, profile_name=profile_name), OriginalEC2Provider(), BotoProvider(), InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher( timeout=metadata_timeout, num_attempts=num_attempts)) ] resolver = CredentialResolver(providers=providers) return resolver
class AutoRefreshableSession: METHOD = "iam-role" DEFAULT_RETRIES = 5 DEFAULT_METADATA_SERVICE_TIMEOUT = 10 # secs def __init__(self, retries=DEFAULT_RETRIES, metadata_service_timeout=DEFAULT_METADATA_SERVICE_TIMEOUT): self.instance_metadata_fetcher = InstanceMetadataFetcher() self.instance_metadata_fetcher._num_attempts = retries self.instance_metadata_fetcher._timeout = metadata_service_timeout self.instance_metadata_fetcher._needs_retry_for_credentials = self.needs_retry_for_credentials self.instance_metadata_provider = InstanceMetadataProvider( self.instance_metadata_fetcher) def check_for_missing_keys(self, required_cred_fields, response): print(response.content) credentials = json.loads(response.content) for field in required_cred_fields: if field not in credentials: print('Retrieved credentials is missing required field: %s', field) return True return False def needs_retry_for_credentials(self, response): return (self.instance_metadata_fetcher._is_non_ok_response(response) or self.instance_metadata_fetcher._is_empty(response) or self.instance_metadata_fetcher._is_invalid_json(response) or self.check_for_missing_keys( self.instance_metadata_fetcher._REQUIRED_CREDENTIAL_FIELDS, response)) def _get(self, region): self.session = get_session() self.session._credentials = self.instance_metadata_provider.load() self.session.set_config_variable("region", region) self.autorefresh_session = Session(botocore_session=self.session) return self def client(self, service_name): return self.autorefresh_session.client(service_name=service_name)
def __call__(self): logger.info('Fetch ec2 instance credentials.') fetcher = InstanceMetadataFetcher( timeout=5.0, num_attempts=10 ) response = fetcher.retrieve_iam_role_credentials() if not response: # There's no way to know exactly reason, # because botocore doesn't pass back the response. # What only we can do here is raise a general error. raise AWSCredentialsError('Retrieve ec2 instance credentials failed.') key_id = response['access_key'] secret_key = response['secret_key'] token = response['token'] expiration = response['expiry_time'] if not isinstance(expiration, datetime): expiration = parse_datetime(expiration) return AWSRawCredentials(key_id, secret_key, token, expiration)
def _aws_credentials_available_in_metadata_service(): import botocore from botocore.credentials import InstanceMetadataProvider from botocore.utils import InstanceMetadataFetcher session = botocore.session.Session() instance_metadata_provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher( timeout=session.get_config_variable('metadata_service_timeout'), num_attempts=session.get_config_variable( 'metadata_service_num_attempts'), user_agent=session.user_agent())) return not (instance_metadata_provider.load() is None)
def credentials(file_loc): if os.path.isfile(file_loc): # Delete any Credentials in the specified file location in order to write new credentials file = open(file_loc,"r+") file.truncate(0) file.close() now = datetime.datetime.utcnow() # Get UTC Time Now provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() file_contents = {'access_key':creds.access_key, 'secret_key':creds.secret_key, 'token':creds.token } file_write(file_loc, file_contents) #Write Tokens to file token_issue_time= datetime.datetime.strptime(now.strftime("%y/%m/%d %H:%M:%S"), "%y/%m/%d %H:%M:%S" ) # Remove time awareness token_issue_expiry = datetime.datetime.strptime((creds._expiry_time).strftime("%y/%m/%d %H:%M:%S"), "%y/%m/%d %H:%M:%S" ) # Time awa expiry_time_sec = (token_issue_expiry - token_issue_time).total_seconds() # Total Seconds before token expiry expiry_time_min = expiry_time_sec/60 return (int(expiry_time_min - 15))
def get_boto_session(): if using_IAM_role: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) print( "Loading IAM Role credentials... (if this is taking a while you are probably not running inside EC2)" ) creds = provider.load() print("IAM credentials loaded") return boto3.Session(aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token, region_name='us-east-1') else: return boto3.Session(region_name='us-east-1')
def create_credential_resolver(session): """Create a default credential resolver. This creates a pre-configured credential resolver that includes the default lookup chain for credentials. """ profile_name = session.get_config_variable('profile') or 'default' credential_file = session.get_config_variable('credentials_file') config_file = session.get_config_variable('config_file') metadata_timeout = session.get_config_variable('metadata_service_timeout') num_attempts = session.get_config_variable('metadata_service_num_attempts') providers = [ SharedCredentialProvider(creds_filename=credential_file, profile_name=profile_name), # The new config file has precedence over the legacy # config file. ConfigProvider(config_filename=config_file, profile_name=profile_name), OriginalEC2Provider(), BotoProvider(), InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher( timeout=metadata_timeout, num_attempts=num_attempts)) ] # We use ``session.profile`` for EnvProvider rather than # ``profile_name`` because it is ``None`` when unset. # TODO: Remove use of internal var. We'll need to rethink this. if session._profile is None: # No profile has been explicitly set, so we prepend the environment # variable provider. That provider, in turn, may set a profile # or credentials. providers.insert(0, EnvProvider()) else: logger.debug('Skipping environment variable credential check' ' because profile name was explicitly set.') resolver = CredentialResolver(providers=providers) return resolver
def handle_ingest(args): """ awspx ingest """ session = None # Get credentials from environment variables if args.env: session = boto3.session.Session(region_name=args.region) # Use existing profile elif args.profile in Profile().credentials.sections(): session = boto3.session.Session(profile_name=args.profile, region_name=args.region) # Use instance profile elif args.profile == "default": try: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher()) creds = provider.load() session = boto3.session.Session( region_name=args.region, aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) except: pass # Specified profile doesn't exist, offer to create it if not session: profile = console.item("Create profile") profile.notice(f"The profile '{args.profile}' doesn't exist. " "Please enter your AWS credentials.\n" "(this information will be saved automatically)") args.create_profile = args.profile handle_profile(args, console=profile) session = boto3.session.Session(profile_name=args.profile, region_name=args.region) # Ancillary operations try: if args.mfa_device: session_token = session.client('sts').get_session_token( SerialNumber=args.mfa_device, TokenCode=args.mfa_token, DurationSeconds=args.mfa_duration)["Credentials"] session = boto3.session.Session( aws_access_key_id=session_token["AccessKeyId"], aws_secret_access_key=session_token["SecretAccessKey"], aws_session_token=session_token["SessionToken"], region_name=args.region) if args.role_to_assume: assume_role_args = { "RoleArn": args.role_to_assume, "RoleSessionName": "awspx", "DurationSeconds": args.role_to_assume_duration, **dict({ "ExternalId": args.role_to_assume_external_id } if args.role_to_assume_external_id else {}) } assumed_role = session.client('sts').assume_role( **assume_role_args)["Credentials"] session = boto3.session.Session( aws_access_key_id=assumed_role["AccessKeyId"], aws_secret_access_key=assumed_role["SecretAccessKey"], aws_session_token=assumed_role["SessionToken"], region_name=args.region) except ClientError as e: console.critical(e) ingestor = IngestionManager(session=session, console=console, services=args.services, db=args.database, quick=args.quick, skip_actions=args.skip_actions_all, only_types=args.only_types, skip_types=args.skip_types, only_arns=args.only_arns, skip_arns=args.skip_arns) assert ingestor.zip is not None, "Ingestion failed" args.load_zips = [ingestor.zip] handle_db(args, console=console.item("Creating Database")) if not (args.skip_attacks_all or args.skip_actions_all): handle_attacks(args, console=console.item("Updating Attack paths"))
def index(): if request.method == 'GET': return 'OK' elif request.method == 'POST': # Store the IP address of the requester request_ip = ipaddress.ip_address(u'{0}'.format(request.remote_addr)) # If VALIDATE_SOURCEIP is set to false, do not validate source IP if os.environ.get('VALIDATE_SOURCEIP', None) != 'false': # If GHE_ADDRESS is specified, use it as the hook_blocks. if os.environ.get('GHE_ADDRESS', None): hook_blocks = [unicode(os.environ.get('GHE_ADDRESS'))] # Otherwise get the hook address blocks from the API. else: hook_blocks = requests.get( 'https://api.github.com/meta').json()['hooks'] # Check if the POST request is from github.com or GHE for block in hook_blocks: if ipaddress.ip_address(request_ip) in ipaddress.ip_network( block): break # the remote_addr is within the network range of github. else: if str(request_ip) != '127.0.0.1': abort(403) if request.headers.get('X-GitHub-Event') == "ping": return json.dumps({'msg': 'Hi!'}) if request.headers.get('X-GitHub-Event') != "push": return json.dumps({'msg': "wrong event type"}) repos = json.loads(io.open(REPOS_JSON_PATH, 'r').read()) payload = json.loads(request.data) repo_meta = { 'name': payload['repository']['name'], 'owner': payload['repository']['owner']['name'], } # Try to match on branch as configured in repos.json match = re.match(r"refs/heads/(?P<branch>.*)", payload['ref']) if match: repo_meta['branch'] = match.groupdict()['branch'] repo = repos.get( '{owner}/{name}/branch:{branch}'.format(**repo_meta), None) # Fallback to plain owner/name lookup if not repo: repo = repos.get('{owner}/{name}'.format(**repo_meta), None) if repo and repo.get('path', None): # Check if POST request signature is valid key = repo.get('key', None) if key: signature = request.headers.get('X-Hub-Signature').split( '=')[1] if type(key) == unicode: key = key.encode() mac = hmac.new(key, msg=request.data, digestmod=sha1) if not compare_digest(mac.hexdigest(), signature): abort(403) if repo.get('action', None): for action in repo['action']: subp = subprocess.Popen(action, cwd=repo.get('path', '.')) subp.wait() if repo.get('s3bucket', None): s3bucketname = repo.get('s3bucket') else: print('missing s3 bucketname') abort(500) if repo.get('s3key', None): s3key = repo.get('s3key') else: print('missing s3 filename') abort(500) print('s3 connection') if os.environ.get('USE_EC2', None) == 'true': provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() session = boto3.Session(aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) s3 = session.resource('s3').Bucket(s3bucketname) else: s3 = boto3.resource('s3') bucket = s3.Bucket(s3bucketname) json.load_s3 = lambda f: json.load(bucket.Object(key=f).get()['Body']) json.dump_s3 = lambda obj, f: bucket.Object(key=f).put(Body=json.dumps( obj)) #s3 fetch s3data = json.load_s3(s3key) datad = FilehashMap(s3data) commithash = payload['after'] for commit in payload['commits']: for z in commit['added']: print(z) datad.additem(z, commithash) for z in commit['modified']: print(z) datad.additem(z, commithash) for z in commit['removed']: datad.delitem(z) print(z) print('s3 upload') json.dump_s3(datad.displayhashmap(), s3key) #set perms s3objacl = s3.ObjectAcl(s3bucketname, s3key) response = s3objacl.put(ACL='public-read') print('s3 done') return 'OK'
def handle_ingest(args): """ awspx ingest """ resources = Elements() account = "000000000000" session = None graph = None # Check to see if environment variables are being used for credentials. if args.env: session = boto3.session.Session(region_name=args.region) # Use existing profile elif args.profile in CREDENTIALS.sections(): session = boto3.session.Session(region_name=args.region, profile_name=args.profile) # Use instance profile elif args.profile == "default": try: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher()) creds = provider.load() session = boto3.session.Session(region_name=args.region, aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) except: pass # Create new profile if not session: if input(f"[-] Would you like to create the profile '{args.profile}'? (y/n) ").upper() == "Y": args.create_profile = args.profile handle_profile(args) session = boto3.session.Session(region_name=args.region, profile_name=args.profile) else: sys.exit(1) try: identity = session.client('sts').get_caller_identity() account = identity["Account"] print(f"[+] Profile: {args.profile} (identity: {identity['Arn']})") except: print("[-] Request to establish identity (sts:GetCallerIdentity) failed.") sys.exit(1) print(f"[+] Services: {', '.join([s.__name__ for s in args.services])}") print(f"[+] Database: {args.database}") print(f"[+] Region: {args.region}") if args.role_to_assume: try: response = session.client('sts').assume_role( RoleArn=args.role_to_assume, RoleSessionName=f"awspx", DurationSeconds=args.role_to_assume_duration) except ClientError as e: print("\n" + str(e)) if "MaxSessionDuration" in e.response["Error"]["Message"]: print("\nTry reducing the session duration using " "'--assume-role-duration'.") sys.exit(1) if response: print(f"[+] Assumed role: {args.role_to_assume}") session = boto3.session.Session( aws_access_key_id=response["Credentials"]["AccessKeyId"], aws_secret_access_key=response["Credentials"]["SecretAccessKey"], aws_session_token=response["Credentials"]["SessionToken"], region_name=args.region) try: identity = session.client('sts').get_caller_identity() account = identity["Account"] print(f"[+] Running as {identity['Arn']}.") print(f"[+] Region set to {args.region}.") except: print("[-] Request to establish identity (sts:GetCallerIdentity) failed.") print() if session is None: sys.exit(1) # Run IAM first to try acquire an account number if IAM in args.services: graph = IAM(session, db=args.database, verbose=args.verbose, quick=args.quick, only_types=args.only_types, skip_types=args.skip_types, only_arns=args.only_arns, skip_arns=args.skip_arns) account = graph.account_id for service in [s for s in args.services if s != IAM]: resources += service(session, account=account, verbose=args.verbose, quick=args.quick, only_types=args.only_types, skip_types=args.skip_types, only_arns=args.only_arns, skip_arns=args.skip_arns) if graph is None: graph = IAM(session, verbose=args.verbose, quick=args.quick, db=args.database, resources=resources) else: graph.update(resources) args.load_zip = graph.post(skip_all_actions=args.skip_all_actions) handle_db(args) if not (args.skip_all_attacks or args.skip_all_actions): handle_attacks(args)