def _load_source_credentials(cls, account, region_name):
key_id = account.access_key_id
secret_key = account.secret_access_key
token = None
expiration = None
if account.is_ec2_instance_role:
logger.info('fetch ec2 instance credentials')
fetcher = InstanceMetadataFetcher(timeout=5.0, num_attempts=10)
response = fetcher.retrieve_iam_role_credentials()
key_id = response['access_key']
secret_key = response['secret_key']
token = response['token']
expiration = response['expiry_time']
if not isinstance(expiration, datetime):
expiration = parse_datetime(expiration)
sts = cls.create_sts_client(key_id, secret_key, token, region_name)
identity = sts.get_caller_identity()
arn = identity.get('Arn')
credentials = AWSCredentials()
credentials.aws_access_key_id = key_id
credentials.aws_secret_access_key = secret_key
credentials.aws_session_token = token
credentials.expiration = expiration
credentials.category = account.category
credentials.arn = arn
return credentials
def get_instance_metadata(self, metadata_type):
"""This method fetches instance metadata using AWS metadata api
Args:
metadata_type (str): Metadata to fetch
Returns:
metadata (str): metadata information for the requested metadata key or
raises a runtime exception.
"""
if metadata_type in ["mac", "instance-id", "security-groups"]:
return urlopen(
os.path.join(self.INSTANCE_METADATA_API, metadata_type),
timeout=self.METADATA_API_TIMEOUT_SECONDS).read().decode(
'utf-8')
elif metadata_type in ["vpc-id", "subnet-id"]:
mac = self.get_instance_metadata("mac")
return urlopen(
os.path.join(self.NETWORK_METADATA_API, mac, metadata_type),
timeout=self.METADATA_API_TIMEOUT_SECONDS).read().decode(
'utf-8')
elif metadata_type in ["region", "privateIp"]:
identity_data = urlopen(self.INSTANCE_IDENTITY_API,
timeout=self.METADATA_API_TIMEOUT_SECONDS) \
.read().decode('utf-8')
return json.loads(identity_data).get(
metadata_type) if identity_data else None
elif metadata_type in ["role"]:
# Arg timeout is in MS.
fetcher = InstanceMetadataFetcher(
timeout=self.METADATA_API_TIMEOUT_SECONDS, num_attempts=2)
c = fetcher.retrieve_iam_role_credentials()
# This will return None in case of no assigned role on the instance.
return c.get("role_name")
else:
raise YBOpsRuntimeError(
"Unsupported metadata type: {}".format(metadata_type))
def test_catch_invalid_imds_error(self):
with mock.patch('botocore.httpsession.URLLib3Session.send') as send_mock:
fetcher = InstanceMetadataFetcher()
e = LocationParseError(location="foo")
send_mock.side_effect = HTTPClientError(error=e)
with self.assertRaises(InvalidIMDSEndpointError):
fetcher.retrieve_iam_role_credentials()
def __init__(self,
retries=DEFAULT_RETRIES,
metadata_service_timeout=DEFAULT_METADATA_SERVICE_TIMEOUT):
self.instance_metadata_fetcher = InstanceMetadataFetcher()
self.instance_metadata_fetcher._num_attempts = retries
self.instance_metadata_fetcher._timeout = metadata_service_timeout
self.instance_metadata_fetcher._needs_retry_for_credentials = self.needs_retry_for_credentials
self.instance_metadata_provider = InstanceMetadataProvider(
self.instance_metadata_fetcher)
def test_catch_retryable_http_errors(self):
with mock.patch('botocore.httpsession.URLLib3Session.send') as send_mock:
fetcher = InstanceMetadataFetcher()
send_mock.side_effect = ConnectionClosedError(endpoint_url="foo")
creds = fetcher.retrieve_iam_role_credentials()
self.assertEqual(send_mock.call_count, 2)
for call_instance in send_mock.call_args_list:
self.assertTrue(call_instance[0][0].url.startswith(fetcher.get_base_url()))
self.assertEqual(creds, {})
def load(self):
timeout = self.session.get_config_variable('metadata_service_timeout')
num_attempts = self.session.get_config_variable(
'metadata_service_num_attempts')
retrieve_kwargs = {}
if timeout is not None:
retrieve_kwargs['timeout'] = float(timeout)
if num_attempts is not None:
retrieve_kwargs['num_attempts'] = int(num_attempts)
fetcher = InstanceMetadataFetcher()
# Partially apply the arguments for future calls.
refresh_using = functools.partial(
fetcher.retrieve_iam_role_credentials, **retrieve_kwargs)
# We do the first request, to see if we get useful data back.
# If not, we'll pass & move on to whatever's next in the credential
# chain.
metadata = refresh_using()
if not metadata:
return None
logger.info('Found IAM Role: %s', metadata['role_name'])
# We manually set the data here, since we already made the request &
# have it. When the expiry is hit, the credentials will auto-refresh
# themselves.
creds = RefreshableCredentials.create_from_metadata(
metadata, method=self.method, refresh_using=refresh_using)
return creds
def create_credential_resolver(session):
"""Create a default credential resolver.
This creates a pre-configured credential resolver
that includes the default lookup chain for
credentials.
"""
profile_name = session.get_config_variable('profile') or 'default'
credential_file = session.get_config_variable('credentials_file')
config_file = session.get_config_variable('config_file')
metadata_timeout = session.get_config_variable('metadata_service_timeout')
num_attempts = session.get_config_variable('metadata_service_num_attempts')
env_provider = EnvProvider()
providers = [
env_provider,
AssumeRoleProvider(
load_config=lambda: session.full_config,
client_creator=session.create_client,
cache={},
profile_name=profile_name,
),
SharedCredentialProvider(creds_filename=credential_file,
profile_name=profile_name),
# The new config file has precedence over the legacy
# config file.
ConfigProvider(config_filename=config_file, profile_name=profile_name),
OriginalEC2Provider(),
BotoProvider(),
ContainerProvider(),
InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(
timeout=metadata_timeout, num_attempts=num_attempts))
]
explicit_profile = session.get_config_variable('profile',
methods=('instance', ))
if explicit_profile is not None:
# An explicitly provided profile will negate an EnvProvider.
# We will defer to providers that understand the "profile"
# concept to retrieve credentials.
# The one edge case if is all three values are provided via
# env vars:
# export AWS_ACCESS_KEY_ID=foo
# export AWS_SECRET_ACCESS_KEY=bar
# export AWS_PROFILE=baz
# Then, just like our client() calls, the explicit credentials
# will take precedence.
#
# This precedence is enforced by leaving the EnvProvider in the chain.
# This means that the only way a "profile" would win is if the
# EnvProvider does not return credentials, which is what we want
# in this scenario.
providers.remove(env_provider)
else:
logger.debug('Skipping environment variable credential check'
' because profile name was explicitly set.')
resolver = CredentialResolver(providers=providers)
return resolver
def create_credential_resolver(session):
"""Create a default credential resolver.
This creates a pre-configured credential resolver
that includes the default lookup chain for
credentials.
"""
profile_name = session.get_config_variable('profile') or 'default'
credential_file = session.get_config_variable('credentials_file')
config_file = session.get_config_variable('config_file')
metadata_timeout = session.get_config_variable('metadata_service_timeout')
num_attempts = session.get_config_variable('metadata_service_num_attempts')
providers = [
EnvProvider(),
SharedCredentialProvider(creds_filename=credential_file,
profile_name=profile_name),
# The new config file has precedence over the legacy
# config file.
ConfigProvider(config_filename=config_file, profile_name=profile_name),
OriginalEC2Provider(),
BotoProvider(),
InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(
timeout=metadata_timeout, num_attempts=num_attempts))
]
resolver = CredentialResolver(providers=providers)
return resolver
class AutoRefreshableSession:
METHOD = "iam-role"
DEFAULT_RETRIES = 5
DEFAULT_METADATA_SERVICE_TIMEOUT = 10 # secs
def __init__(self,
retries=DEFAULT_RETRIES,
metadata_service_timeout=DEFAULT_METADATA_SERVICE_TIMEOUT):
self.instance_metadata_fetcher = InstanceMetadataFetcher()
self.instance_metadata_fetcher._num_attempts = retries
self.instance_metadata_fetcher._timeout = metadata_service_timeout
self.instance_metadata_fetcher._needs_retry_for_credentials = self.needs_retry_for_credentials
self.instance_metadata_provider = InstanceMetadataProvider(
self.instance_metadata_fetcher)
def check_for_missing_keys(self, required_cred_fields, response):
print(response.content)
credentials = json.loads(response.content)
for field in required_cred_fields:
if field not in credentials:
print('Retrieved credentials is missing required field: %s',
field)
return True
return False
def needs_retry_for_credentials(self, response):
return (self.instance_metadata_fetcher._is_non_ok_response(response)
or self.instance_metadata_fetcher._is_empty(response)
or self.instance_metadata_fetcher._is_invalid_json(response)
or self.check_for_missing_keys(
self.instance_metadata_fetcher._REQUIRED_CREDENTIAL_FIELDS,
response))
def _get(self, region):
self.session = get_session()
self.session._credentials = self.instance_metadata_provider.load()
self.session.set_config_variable("region", region)
self.autorefresh_session = Session(botocore_session=self.session)
return self
def client(self, service_name):
return self.autorefresh_session.client(service_name=service_name)
def __call__(self):
logger.info('Fetch ec2 instance credentials.')
fetcher = InstanceMetadataFetcher(
timeout=5.0,
num_attempts=10
)
response = fetcher.retrieve_iam_role_credentials()
if not response:
# There's no way to know exactly reason,
# because botocore doesn't pass back the response.
# What only we can do here is raise a general error.
raise AWSCredentialsError('Retrieve ec2 instance credentials failed.')
key_id = response['access_key']
secret_key = response['secret_key']
token = response['token']
expiration = response['expiry_time']
if not isinstance(expiration, datetime):
expiration = parse_datetime(expiration)
return AWSRawCredentials(key_id, secret_key, token, expiration)
def _aws_credentials_available_in_metadata_service():
import botocore
from botocore.credentials import InstanceMetadataProvider
from botocore.utils import InstanceMetadataFetcher
session = botocore.session.Session()
instance_metadata_provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher(
timeout=session.get_config_variable('metadata_service_timeout'),
num_attempts=session.get_config_variable(
'metadata_service_num_attempts'),
user_agent=session.user_agent()))
return not (instance_metadata_provider.load() is None)
def credentials(file_loc):
if os.path.isfile(file_loc): # Delete any Credentials in the specified file location in order to write new credentials
file = open(file_loc,"r+")
file.truncate(0)
file.close()
now = datetime.datetime.utcnow() # Get UTC Time Now
provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2))
creds = provider.load()
file_contents = {'access_key':creds.access_key, 'secret_key':creds.secret_key, 'token':creds.token }
file_write(file_loc, file_contents) #Write Tokens to file
token_issue_time= datetime.datetime.strptime(now.strftime("%y/%m/%d %H:%M:%S"), "%y/%m/%d %H:%M:%S" ) # Remove time awareness
token_issue_expiry = datetime.datetime.strptime((creds._expiry_time).strftime("%y/%m/%d %H:%M:%S"), "%y/%m/%d %H:%M:%S" ) # Time awa
expiry_time_sec = (token_issue_expiry - token_issue_time).total_seconds() # Total Seconds before token expiry
expiry_time_min = expiry_time_sec/60
return (int(expiry_time_min - 15))
def get_boto_session():
if using_IAM_role:
provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher(timeout=1000,
num_attempts=2))
print(
"Loading IAM Role credentials... (if this is taking a while you are probably not running inside EC2)"
)
creds = provider.load()
print("IAM credentials loaded")
return boto3.Session(aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token,
region_name='us-east-1')
else:
return boto3.Session(region_name='us-east-1')
def create_credential_resolver(session):
"""Create a default credential resolver.
This creates a pre-configured credential resolver
that includes the default lookup chain for
credentials.
"""
profile_name = session.get_config_variable('profile') or 'default'
credential_file = session.get_config_variable('credentials_file')
config_file = session.get_config_variable('config_file')
metadata_timeout = session.get_config_variable('metadata_service_timeout')
num_attempts = session.get_config_variable('metadata_service_num_attempts')
providers = [
SharedCredentialProvider(creds_filename=credential_file,
profile_name=profile_name),
# The new config file has precedence over the legacy
# config file.
ConfigProvider(config_filename=config_file, profile_name=profile_name),
OriginalEC2Provider(),
BotoProvider(),
InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(
timeout=metadata_timeout, num_attempts=num_attempts))
]
# We use ``session.profile`` for EnvProvider rather than
# ``profile_name`` because it is ``None`` when unset.
# TODO: Remove use of internal var. We'll need to rethink this.
if session._profile is None:
# No profile has been explicitly set, so we prepend the environment
# variable provider. That provider, in turn, may set a profile
# or credentials.
providers.insert(0, EnvProvider())
else:
logger.debug('Skipping environment variable credential check'
' because profile name was explicitly set.')
resolver = CredentialResolver(providers=providers)
return resolver
def handle_ingest(args):
"""
awspx ingest
"""
session = None
# Get credentials from environment variables
if args.env:
session = boto3.session.Session(region_name=args.region)
# Use existing profile
elif args.profile in Profile().credentials.sections():
session = boto3.session.Session(profile_name=args.profile,
region_name=args.region)
# Use instance profile
elif args.profile == "default":
try:
provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher())
creds = provider.load()
session = boto3.session.Session(
region_name=args.region,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token)
except:
pass
# Specified profile doesn't exist, offer to create it
if not session:
profile = console.item("Create profile")
profile.notice(f"The profile '{args.profile}' doesn't exist. "
"Please enter your AWS credentials.\n"
"(this information will be saved automatically)")
args.create_profile = args.profile
handle_profile(args, console=profile)
session = boto3.session.Session(profile_name=args.profile,
region_name=args.region)
# Ancillary operations
try:
if args.mfa_device:
session_token = session.client('sts').get_session_token(
SerialNumber=args.mfa_device,
TokenCode=args.mfa_token,
DurationSeconds=args.mfa_duration)["Credentials"]
session = boto3.session.Session(
aws_access_key_id=session_token["AccessKeyId"],
aws_secret_access_key=session_token["SecretAccessKey"],
aws_session_token=session_token["SessionToken"],
region_name=args.region)
if args.role_to_assume:
assume_role_args = {
"RoleArn":
args.role_to_assume,
"RoleSessionName":
"awspx",
"DurationSeconds":
args.role_to_assume_duration,
**dict({
"ExternalId": args.role_to_assume_external_id
} if args.role_to_assume_external_id else {})
}
assumed_role = session.client('sts').assume_role(
**assume_role_args)["Credentials"]
session = boto3.session.Session(
aws_access_key_id=assumed_role["AccessKeyId"],
aws_secret_access_key=assumed_role["SecretAccessKey"],
aws_session_token=assumed_role["SessionToken"],
region_name=args.region)
except ClientError as e:
console.critical(e)
ingestor = IngestionManager(session=session,
console=console,
services=args.services,
db=args.database,
quick=args.quick,
skip_actions=args.skip_actions_all,
only_types=args.only_types,
skip_types=args.skip_types,
only_arns=args.only_arns,
skip_arns=args.skip_arns)
assert ingestor.zip is not None, "Ingestion failed"
args.load_zips = [ingestor.zip]
handle_db(args, console=console.item("Creating Database"))
if not (args.skip_attacks_all or args.skip_actions_all):
handle_attacks(args, console=console.item("Updating Attack paths"))
def index():
if request.method == 'GET':
return 'OK'
elif request.method == 'POST':
# Store the IP address of the requester
request_ip = ipaddress.ip_address(u'{0}'.format(request.remote_addr))
# If VALIDATE_SOURCEIP is set to false, do not validate source IP
if os.environ.get('VALIDATE_SOURCEIP', None) != 'false':
# If GHE_ADDRESS is specified, use it as the hook_blocks.
if os.environ.get('GHE_ADDRESS', None):
hook_blocks = [unicode(os.environ.get('GHE_ADDRESS'))]
# Otherwise get the hook address blocks from the API.
else:
hook_blocks = requests.get(
'https://api.github.com/meta').json()['hooks']
# Check if the POST request is from github.com or GHE
for block in hook_blocks:
if ipaddress.ip_address(request_ip) in ipaddress.ip_network(
block):
break # the remote_addr is within the network range of github.
else:
if str(request_ip) != '127.0.0.1':
abort(403)
if request.headers.get('X-GitHub-Event') == "ping":
return json.dumps({'msg': 'Hi!'})
if request.headers.get('X-GitHub-Event') != "push":
return json.dumps({'msg': "wrong event type"})
repos = json.loads(io.open(REPOS_JSON_PATH, 'r').read())
payload = json.loads(request.data)
repo_meta = {
'name': payload['repository']['name'],
'owner': payload['repository']['owner']['name'],
}
# Try to match on branch as configured in repos.json
match = re.match(r"refs/heads/(?P<branch>.*)", payload['ref'])
if match:
repo_meta['branch'] = match.groupdict()['branch']
repo = repos.get(
'{owner}/{name}/branch:{branch}'.format(**repo_meta), None)
# Fallback to plain owner/name lookup
if not repo:
repo = repos.get('{owner}/{name}'.format(**repo_meta), None)
if repo and repo.get('path', None):
# Check if POST request signature is valid
key = repo.get('key', None)
if key:
signature = request.headers.get('X-Hub-Signature').split(
'=')[1]
if type(key) == unicode:
key = key.encode()
mac = hmac.new(key, msg=request.data, digestmod=sha1)
if not compare_digest(mac.hexdigest(), signature):
abort(403)
if repo.get('action', None):
for action in repo['action']:
subp = subprocess.Popen(action, cwd=repo.get('path', '.'))
subp.wait()
if repo.get('s3bucket', None):
s3bucketname = repo.get('s3bucket')
else:
print('missing s3 bucketname')
abort(500)
if repo.get('s3key', None):
s3key = repo.get('s3key')
else:
print('missing s3 filename')
abort(500)
print('s3 connection')
if os.environ.get('USE_EC2', None) == 'true':
provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher(timeout=1000,
num_attempts=2))
creds = provider.load()
session = boto3.Session(aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token)
s3 = session.resource('s3').Bucket(s3bucketname)
else:
s3 = boto3.resource('s3')
bucket = s3.Bucket(s3bucketname)
json.load_s3 = lambda f: json.load(bucket.Object(key=f).get()['Body'])
json.dump_s3 = lambda obj, f: bucket.Object(key=f).put(Body=json.dumps(
obj))
#s3 fetch
s3data = json.load_s3(s3key)
datad = FilehashMap(s3data)
commithash = payload['after']
for commit in payload['commits']:
for z in commit['added']:
print(z)
datad.additem(z, commithash)
for z in commit['modified']:
print(z)
datad.additem(z, commithash)
for z in commit['removed']:
datad.delitem(z)
print(z)
print('s3 upload')
json.dump_s3(datad.displayhashmap(), s3key)
#set perms
s3objacl = s3.ObjectAcl(s3bucketname, s3key)
response = s3objacl.put(ACL='public-read')
print('s3 done')
return 'OK'
def handle_ingest(args):
"""
awspx ingest
"""
resources = Elements()
account = "000000000000"
session = None
graph = None
# Check to see if environment variables are being used for credentials.
if args.env:
session = boto3.session.Session(region_name=args.region)
# Use existing profile
elif args.profile in CREDENTIALS.sections():
session = boto3.session.Session(region_name=args.region,
profile_name=args.profile)
# Use instance profile
elif args.profile == "default":
try:
provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher())
creds = provider.load()
session = boto3.session.Session(region_name=args.region,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token)
except:
pass
# Create new profile
if not session:
if input(f"[-] Would you like to create the profile '{args.profile}'? (y/n) ").upper() == "Y":
args.create_profile = args.profile
handle_profile(args)
session = boto3.session.Session(region_name=args.region,
profile_name=args.profile)
else:
sys.exit(1)
try:
identity = session.client('sts').get_caller_identity()
account = identity["Account"]
print(f"[+] Profile: {args.profile} (identity: {identity['Arn']})")
except:
print("[-] Request to establish identity (sts:GetCallerIdentity) failed.")
sys.exit(1)
print(f"[+] Services: {', '.join([s.__name__ for s in args.services])}")
print(f"[+] Database: {args.database}")
print(f"[+] Region: {args.region}")
if args.role_to_assume:
try:
response = session.client('sts').assume_role(
RoleArn=args.role_to_assume,
RoleSessionName=f"awspx",
DurationSeconds=args.role_to_assume_duration)
except ClientError as e:
print("\n" + str(e))
if "MaxSessionDuration" in e.response["Error"]["Message"]:
print("\nTry reducing the session duration using "
"'--assume-role-duration'.")
sys.exit(1)
if response:
print(f"[+] Assumed role: {args.role_to_assume}")
session = boto3.session.Session(
aws_access_key_id=response["Credentials"]["AccessKeyId"],
aws_secret_access_key=response["Credentials"]["SecretAccessKey"],
aws_session_token=response["Credentials"]["SessionToken"],
region_name=args.region)
try:
identity = session.client('sts').get_caller_identity()
account = identity["Account"]
print(f"[+] Running as {identity['Arn']}.")
print(f"[+] Region set to {args.region}.")
except:
print("[-] Request to establish identity (sts:GetCallerIdentity) failed.")
print()
if session is None:
sys.exit(1)
# Run IAM first to try acquire an account number
if IAM in args.services:
graph = IAM(session, db=args.database, verbose=args.verbose, quick=args.quick,
only_types=args.only_types, skip_types=args.skip_types,
only_arns=args.only_arns, skip_arns=args.skip_arns)
account = graph.account_id
for service in [s for s in args.services if s != IAM]:
resources += service(session, account=account, verbose=args.verbose, quick=args.quick,
only_types=args.only_types, skip_types=args.skip_types,
only_arns=args.only_arns, skip_arns=args.skip_arns)
if graph is None:
graph = IAM(session, verbose=args.verbose, quick=args.quick,
db=args.database,
resources=resources)
else:
graph.update(resources)
args.load_zip = graph.post(skip_all_actions=args.skip_all_actions)
handle_db(args)
if not (args.skip_all_attacks or args.skip_all_actions):
handle_attacks(args)
