def getCredentials(self): provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() access_key = creds.access_key return (access_key)
def __init__(self, dest=None, content_type=None): self.errors = [] if not dest: dest = 'default' try: destination_settings =\ app_settings.S3DIRECT_DESTINATIONS[dest] except KeyError: self.errors.append( 'Provided destination is not specified in config', ) return for key, value in destination_settings.items(): setattr(self, key, value) self.access_key =\ getattr(settings, 'AWS_S3_ACCESS_KEY_ID', None) self.secret_access_key =\ getattr(settings, 'AWS_S3_SECRET_ACCESS_KEY', None) self.now_date = datetime.utcnow().strftime('%Y%m%dT%H%M%S000Z') self.raw_date = datetime.utcnow().strftime('%Y%m%d') if content_type: self.content_type = content_type if (content_type and (self.allowed and content_type not in self.allowed) and self.allowed != '*'): self.errors.append('Invalid file type.') return if not self.region or self.region == 'us-east-1': self.endpoint = 's3.amazonaws.com' self.endpoint = f's3-{self.region}.amazonaws.com' if self.access_key is None or self.secret_access_key is None: try: from botocore.credentials import (InstanceMetadataProvider, InstanceMetadataFetcher) except ImportError: InstanceMetadataProvider = None InstanceMetadataFetcher = None if all([InstanceMetadataProvider, InstanceMetadataFetcher]): provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() self.access_key = creds.access_key self.secret_access_key = creds.secret_key self.token = creds.token else: self.errors.append( 'Failed to access EC2 instance metadata due to ' 'missing dependency.') return
def _aws_credentials_available_in_metadata_service(): import botocore from botocore.credentials import InstanceMetadataProvider from botocore.utils import InstanceMetadataFetcher session = botocore.session.Session() instance_metadata_provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher( timeout=session.get_config_variable('metadata_service_timeout'), num_attempts=session.get_config_variable( 'metadata_service_num_attempts'), user_agent=session.user_agent())) return not (instance_metadata_provider.load() is None)
def delete_object(bucket, key): rn = "eu-west-1" provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load().get_frozen_credentials() s3_client = boto3.client( "s3", region_name=rn, aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token, ) s3_client.delete_object(Bucket=bucket, Key=key)
def credentials(file_loc): if os.path.isfile(file_loc): # Delete any Credentials in the specified file location in order to write new credentials file = open(file_loc,"r+") file.truncate(0) file.close() now = datetime.datetime.utcnow() # Get UTC Time Now provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() file_contents = {'access_key':creds.access_key, 'secret_key':creds.secret_key, 'token':creds.token } file_write(file_loc, file_contents) #Write Tokens to file token_issue_time= datetime.datetime.strptime(now.strftime("%y/%m/%d %H:%M:%S"), "%y/%m/%d %H:%M:%S" ) # Remove time awareness token_issue_expiry = datetime.datetime.strptime((creds._expiry_time).strftime("%y/%m/%d %H:%M:%S"), "%y/%m/%d %H:%M:%S" ) # Time awa expiry_time_sec = (token_issue_expiry - token_issue_time).total_seconds() # Total Seconds before token expiry expiry_time_min = expiry_time_sec/60 return (int(expiry_time_min - 15))
def get_boto_session(): if using_IAM_role: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) print( "Loading IAM Role credentials... (if this is taking a while you are probably not running inside EC2)" ) creds = provider.load() print("IAM credentials loaded") return boto3.Session(aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token, region_name='us-east-1') else: return boto3.Session(region_name='us-east-1')
def get_boto_session( force_ec2: bool = False, region_name: str = "eu-west-1", ): kwargs = {"region_name": region_name} if force_ec2: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load().get_frozen_credentials() kwargs["aws_access_key_id"] = creds.access_key kwargs["aws_secret_access_key"] = creds.secret_key kwargs["aws_session_token"] = creds.token return boto3.Session(**kwargs)
def get_aws_credentials(): access_key = getattr(settings, 'AWS_ACCESS_KEY_ID', None) secret_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None) if access_key and secret_key: # AWS tokens are not created for pregenerated access keys return AWSCredentials(None, secret_key, access_key) if not InstanceMetadataProvider or not InstanceMetadataFetcher: # AWS credentials are not required for publicly-writable buckets return AWSCredentials(None, None, None) provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() if creds: return AWSCredentials(creds.token, creds.secret_key, creds.access_key) else: # Creds are incorrect return AWSCredentials(None, None, None)
def get_aws_credentials(): access_key = getattr(settings, 'AWS_ACCESS_KEY_ID', None) secret_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None) if access_key and secret_key: # AWS tokens are not created for pregenerated access keys return AWSCredentials(None, secret_key, access_key) if not InstanceMetadataProvider or not InstanceMetadataFetcher: # AWS credentials are not required for publicly-writable buckets return AWSCredentials(None, None, None) provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() if creds: return AWSCredentials(creds.token, creds.secret_key, creds.access_key) else: # Creds are incorrect return AWSCredentials(None, None, None)
class AutoRefreshableSession: METHOD = "iam-role" DEFAULT_RETRIES = 5 DEFAULT_METADATA_SERVICE_TIMEOUT = 10 # secs def __init__(self, retries=DEFAULT_RETRIES, metadata_service_timeout=DEFAULT_METADATA_SERVICE_TIMEOUT): self.instance_metadata_fetcher = InstanceMetadataFetcher() self.instance_metadata_fetcher._num_attempts = retries self.instance_metadata_fetcher._timeout = metadata_service_timeout self.instance_metadata_fetcher._needs_retry_for_credentials = self.needs_retry_for_credentials self.instance_metadata_provider = InstanceMetadataProvider( self.instance_metadata_fetcher) def check_for_missing_keys(self, required_cred_fields, response): print(response.content) credentials = json.loads(response.content) for field in required_cred_fields: if field not in credentials: print('Retrieved credentials is missing required field: %s', field) return True return False def needs_retry_for_credentials(self, response): return (self.instance_metadata_fetcher._is_non_ok_response(response) or self.instance_metadata_fetcher._is_empty(response) or self.instance_metadata_fetcher._is_invalid_json(response) or self.check_for_missing_keys( self.instance_metadata_fetcher._REQUIRED_CREDENTIAL_FIELDS, response)) def _get(self, region): self.session = get_session() self.session._credentials = self.instance_metadata_provider.load() self.session.set_config_variable("region", region) self.autorefresh_session = Session(botocore_session=self.session) return self def client(self, service_name): return self.autorefresh_session.client(service_name=service_name)
def __call__(self, host, username, password, aws_region, boto_profile): """ Return the authorization header. If 'boto_profile' is passed, it'll be used. Otherwise it'll sign requests with instance role. :param host: ElasticSearch host. :param username: Username used for authenticating the requests to ElasticSearch. :param password: Password used for authenticating the requests to ElasticSearch. :param aws_region: AWS Region to use. Only required when signing requests. :param boto_profile: Boto profile to use for connecting. Only required when signing requests. """ if username and password: return username + ":" + password if not aws_region: return None if boto_profile: # Executing elastalert from machine with aws credentials config = configparser.ConfigParser() config.read(os.path.expanduser("~") + "/.aws/credentials") aws_access_key_id = str(config[boto_profile]["aws_access_key_id"]) aws_secret_access_key = str(config[boto_profile]["aws_secret_access_key"]) aws_token = None else: # Executing elastalert from machine deployed with specific role provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) aws_credentials = provider.load() aws_access_key_id = str(aws_credentials.access_key) aws_secret_access_key = str(aws_credentials.secret_key) aws_token = str(aws_credentials.token) return AWSRequestsAuth( aws_access_key=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_token=aws_token, aws_host=host, aws_region=aws_region, aws_service="es", )
def __call__(self, host, username, password, aws_region, boto_profile): """ Return the authorization header. If 'boto_profile' is passed, it'll be used. Otherwise it'll sign requests with instance role. :param host: ElasticSearch host. :param username: Username used for authenticating the requests to ElasticSearch. :param password: Password used for authenticating the requests to ElasticSearch. :param aws_region: AWS Region to use. Only required when signing requests. :param boto_profile: Boto profile to use for connecting. Only required when signing requests. """ if username and password: return username + ':' + password if not aws_region: return None if boto_profile: # Executing elastalert from machine with aws credentials config = configparser.ConfigParser() config.read(os.path.expanduser('~') + '/.aws/credentials') aws_access_key_id = str(config[boto_profile]['aws_access_key_id']) aws_secret_access_key = str(config[boto_profile]['aws_secret_access_key']) aws_token = None else: # Executing elastalert from machine deployed with specific role provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) aws_credentials = provider.load() aws_access_key_id = str(aws_credentials.access_key) aws_secret_access_key = str(aws_credentials.secret_key) aws_token = str(aws_credentials.token) return AWSRequestsAuth(aws_access_key=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_token=aws_token, aws_host=host, aws_region=aws_region, aws_service='es')
def __setupSparkSession__(self, appName: str = None): ''' Init the Spark environemnt with few default configurations and start the spark session. ''' self.__conf = SparkConf() hmConf = { "spark.rps.askTimeout": "1200", "spark.network.timeout": "1200", "spark.broadcast.blockSize": "16m", "spark.sql.broadcastTimeout": "1200", "spark.broadcast.compress": "true", "spark.rdd.compress": "true", "fs.s3.enableServerSideEncryption": "true", "spark.kryo.unsafe": "false", "spark.kryoserializer.buffer": "10240", "spark.kryoserializer.buffer.max": "2040m", "spark.io.compression.codec": "org.apache.spark.io.SnappyCompressionCodec", "spark.serializer": "org.apache.spark.serializer.KryoSerializer", "mapreduce.fileoutputcommitter.algorithm.version": "2", "spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version": "2", } self.__conf.setAll(hmConf) SparkContext.setSystemProperty("com.amazonaws.services.s3.enableV4", "true") SparkContext.setSystemProperty("com.amazonaws.services.s3.enforceV4", "true") self.__spark = SparkSession \ .builder \ .config(conf=self.__conf) \ .appName(appName or "PySparkApp") \ .enableHiveSupport() \ .getOrCreate() self.__sc = self.__spark.sparkContext self.sqlC = SQLContext(self.__sc) self.__sc.setSystemProperty("com.amazonaws.services.s3.enableV4", "true") self.__sc.setSystemProperty("com.amazonaws.services.s3.enforceV4", "true") self.__sc.setLogLevel(self.__parms.get("--logLevel", "INFO")) hdpCnf = self.__sc.hadoopConfiguration hdpCnf.setAll({ "io.file.buffer.size": "65536", "mapreduce.fileoutputcommitter.algorithm.version": "2", "fs.s3a.endpoint": "%s.amazonaws.com" % (self.__parms.get("--awsRegion", 's3.us-east-1')) }) if (self.__parms.get("--runEnv", "AWS") == "AWS"): from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() hdpCnf.setAll({ "fs.s3a.access.key": creds.access_key, "fs.s3a.secret.key": creds.secret_key, "fs.s3a.server-side-encryption-algorithm": "SSE-KMS", "fs.s3.enableServerSideEncryption": "true", "fs.s3.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", "fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", "fs.s3a.endpoint": "s3.%s.amazonaws.com" % (self.__parms.get("--awsRegion", "us-east-1")) })
def handle_ingest(args): """ awspx ingest """ session = None # Get credentials from environment variables if args.env: session = boto3.session.Session(region_name=args.region) # Use existing profile elif args.profile in Profile().credentials.sections(): session = boto3.session.Session(profile_name=args.profile, region_name=args.region) # Use instance profile elif args.profile == "default": try: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher()) creds = provider.load() session = boto3.session.Session( region_name=args.region, aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) except: pass # Specified profile doesn't exist, offer to create it if not session: profile = console.item("Create profile") profile.notice(f"The profile '{args.profile}' doesn't exist. " "Please enter your AWS credentials.\n" "(this information will be saved automatically)") args.create_profile = args.profile handle_profile(args, console=profile) session = boto3.session.Session(profile_name=args.profile, region_name=args.region) # Ancillary operations try: if args.mfa_device: session_token = session.client('sts').get_session_token( SerialNumber=args.mfa_device, TokenCode=args.mfa_token, DurationSeconds=args.mfa_duration)["Credentials"] session = boto3.session.Session( aws_access_key_id=session_token["AccessKeyId"], aws_secret_access_key=session_token["SecretAccessKey"], aws_session_token=session_token["SessionToken"], region_name=args.region) if args.role_to_assume: assume_role_args = { "RoleArn": args.role_to_assume, "RoleSessionName": "awspx", "DurationSeconds": args.role_to_assume_duration, **dict({ "ExternalId": args.role_to_assume_external_id } if args.role_to_assume_external_id else {}) } assumed_role = session.client('sts').assume_role( **assume_role_args)["Credentials"] session = boto3.session.Session( aws_access_key_id=assumed_role["AccessKeyId"], aws_secret_access_key=assumed_role["SecretAccessKey"], aws_session_token=assumed_role["SessionToken"], region_name=args.region) except ClientError as e: console.critical(e) ingestor = IngestionManager(session=session, console=console, services=args.services, db=args.database, quick=args.quick, skip_actions=args.skip_actions_all, only_types=args.only_types, skip_types=args.skip_types, only_arns=args.only_arns, skip_arns=args.skip_arns) assert ingestor.zip is not None, "Ingestion failed" args.load_zips = [ingestor.zip] handle_db(args, console=console.item("Creating Database")) if not (args.skip_attacks_all or args.skip_actions_all): handle_attacks(args, console=console.item("Updating Attack paths"))
def get_upload_params(request): content_type = request.POST['type'] filename = request.POST['name'] dest = get_s3direct_destinations().get(request.POST['dest']) if not dest: data = json.dumps({'error': 'File destination does not exist.'}) return HttpResponse(data, content_type="application/json", status=400) key = dest.get('key') auth = dest.get('auth') allowed = dest.get('allowed') acl = dest.get('acl') bucket = dest.get('bucket') cache_control = dest.get('cache_control') content_disposition = dest.get('content_disposition') content_length_range = dest.get('content_length_range') server_side_encryption = dest.get('server_side_encryption') if not acl: acl = 'public-read' if not key: data = json.dumps({'error': 'Missing destination path.'}) return HttpResponse(data, content_type="application/json", status=403) if auth and not auth(request.user): data = json.dumps({'error': 'Permission denied.'}) return HttpResponse(data, content_type="application/json", status=403) if (allowed and content_type not in allowed) and allowed != '*': data = json.dumps({'error': 'Invalid file type (%s).' % content_type}) return HttpResponse(data, content_type="application/json", status=400) if hasattr(key, '__call__'): key = key(filename) elif key == '/': key = '${filename}' else: # The literal string '${filename}' is an S3 field variable for key. # https://aws.amazon.com/articles/1434#aws-table key = '%s/${filename}' % key access_key = getattr(settings, 'AWS_ACCESS_KEY_ID', None) secret_access_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None) token = None if access_key is None or secret_access_key is None: # Get credentials from instance profile if not defined in settings -- # this avoids the need to put access credentials in the settings.py file. # Assumes we're running on EC2. try: from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher except ImportError: InstanceMetadataProvider = None InstanceMetadataFetcher = None if all([InstanceMetadataProvider, InstanceMetadataFetcher]): provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() access_key = creds.access_key secret_access_key = creds.secret_key token = creds.token else: data = json.dumps({ 'error': 'Failed to access EC2 instance metadata due to missing dependency.' }) return HttpResponse(data, content_type="application/json", status=500) data = create_upload_data(content_type, key, acl, bucket, cache_control, content_disposition, content_length_range, server_side_encryption, access_key, secret_access_key, token) return HttpResponse(json.dumps(data), content_type="application/json")
def handle_ingest(args): """ awspx ingest """ resources = Elements() account = "000000000000" session = None graph = None # Check to see if environment variables are being used for credentials. if args.env: session = boto3.session.Session(region_name=args.region) # Use existing profile elif args.profile in CREDENTIALS.sections(): session = boto3.session.Session(region_name=args.region, profile_name=args.profile) # Use instance profile elif args.profile == "default": try: provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher()) creds = provider.load() session = boto3.session.Session(region_name=args.region, aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) except: pass # Create new profile if not session: if input(f"[-] Would you like to create the profile '{args.profile}'? (y/n) ").upper() == "Y": args.create_profile = args.profile handle_profile(args) session = boto3.session.Session(region_name=args.region, profile_name=args.profile) else: sys.exit(1) try: identity = session.client('sts').get_caller_identity() account = identity["Account"] print(f"[+] Profile: {args.profile} (identity: {identity['Arn']})") except: print("[-] Request to establish identity (sts:GetCallerIdentity) failed.") sys.exit(1) print(f"[+] Services: {', '.join([s.__name__ for s in args.services])}") print(f"[+] Database: {args.database}") print(f"[+] Region: {args.region}") if args.role_to_assume: try: response = session.client('sts').assume_role( RoleArn=args.role_to_assume, RoleSessionName=f"awspx", DurationSeconds=args.role_to_assume_duration) except ClientError as e: print("\n" + str(e)) if "MaxSessionDuration" in e.response["Error"]["Message"]: print("\nTry reducing the session duration using " "'--assume-role-duration'.") sys.exit(1) if response: print(f"[+] Assumed role: {args.role_to_assume}") session = boto3.session.Session( aws_access_key_id=response["Credentials"]["AccessKeyId"], aws_secret_access_key=response["Credentials"]["SecretAccessKey"], aws_session_token=response["Credentials"]["SessionToken"], region_name=args.region) try: identity = session.client('sts').get_caller_identity() account = identity["Account"] print(f"[+] Running as {identity['Arn']}.") print(f"[+] Region set to {args.region}.") except: print("[-] Request to establish identity (sts:GetCallerIdentity) failed.") print() if session is None: sys.exit(1) # Run IAM first to try acquire an account number if IAM in args.services: graph = IAM(session, db=args.database, verbose=args.verbose, quick=args.quick, only_types=args.only_types, skip_types=args.skip_types, only_arns=args.only_arns, skip_arns=args.skip_arns) account = graph.account_id for service in [s for s in args.services if s != IAM]: resources += service(session, account=account, verbose=args.verbose, quick=args.quick, only_types=args.only_types, skip_types=args.skip_types, only_arns=args.only_arns, skip_arns=args.skip_arns) if graph is None: graph = IAM(session, verbose=args.verbose, quick=args.quick, db=args.database, resources=resources) else: graph.update(resources) args.load_zip = graph.post(skip_all_actions=args.skip_all_actions) handle_db(args) if not (args.skip_all_attacks or args.skip_all_actions): handle_attacks(args)
def get_upload_params(request): content_type = request.POST['type'] filename = get_valid_filename(request.POST['name']) dest = get_s3upload_destinations().get(request.POST['dest']) if not dest: data = json.dumps({'error': 'File destination does not exist.'}) return HttpResponse(data, content_type="application/json", status=400) key = dest.get('key') auth = dest.get('auth') allowed_types = dest.get('allowed_types') acl = dest.get('acl') bucket = dest.get('bucket') cache_control = dest.get('cache_control') content_disposition = dest.get('content_disposition') content_length_range = dest.get('content_length_range') allowed_extensions = dest.get('allowed_extensions') server_side_encryption = dest.get('server_side_encryption') if not acl: acl = 'public-read' if not key: data = json.dumps({'error': 'Missing destination path.'}) return HttpResponse(data, content_type="application/json", status=403) if auth and not auth(request.user): data = json.dumps({'error': 'Permission denied.'}) return HttpResponse(data, content_type="application/json", status=403) if (allowed_types and content_type not in allowed_types) and allowed_types != '*': data = json.dumps({'error': 'Invalid file type (%s).' % content_type}) return HttpResponse(data, content_type="application/json", status=400) original_ext = splitext(filename)[1] lowercased_ext = original_ext.lower() if (allowed_extensions and lowercased_ext not in allowed_extensions) and allowed_extensions != '*': data = json.dumps( {'error': 'Forbidden file extension (%s).' % original_ext}) return HttpResponse(data, content_type="application/json", status=415) if hasattr(key, '__call__'): key = key(filename) elif key == '/': key = filename else: key = '{0}/{1}'.format(key, filename) access_key = getattr(settings, 'AWS_ACCESS_KEY_ID', None) secret_access_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None) token = None if access_key is None or secret_access_key is None: # Get credentials from instance profile if not defined in settings -- # this avoids the need to put access credentials in the settings.py file. # Assumes we're running on EC2. try: from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher except ImportError: InstanceMetadataProvider = None InstanceMetadataFetcher = None if all([InstanceMetadataProvider, InstanceMetadataFetcher]): provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() access_key = creds.access_key secret_access_key = creds.secret_key token = creds.token else: data = json.dumps({ 'error': 'Failed to access EC2 instance metadata due to missing dependency.' }) return HttpResponse(data, content_type="application/json", status=500) data = create_upload_data(content_type, key, acl, bucket, cache_control, content_disposition, content_length_range, server_side_encryption, access_key, secret_access_key, token) url = None # Generate signed URL for private document access if acl == "private": url = get_signed_download_url( key=key.replace("${filename}", filename), bucket_name=bucket or settings.AWS_STORAGE_BUCKET_NAME, ttl=int(5 * 60), # 5 mins ) response = { "aws_payload": data, "private_access_url": url, } return HttpResponse(json.dumps(response), content_type="application/json")
def index(): if request.method == 'GET': return 'OK' elif request.method == 'POST': # Store the IP address of the requester request_ip = ipaddress.ip_address(u'{0}'.format(request.remote_addr)) # If VALIDATE_SOURCEIP is set to false, do not validate source IP if os.environ.get('VALIDATE_SOURCEIP', None) != 'false': # If GHE_ADDRESS is specified, use it as the hook_blocks. if os.environ.get('GHE_ADDRESS', None): hook_blocks = [unicode(os.environ.get('GHE_ADDRESS'))] # Otherwise get the hook address blocks from the API. else: hook_blocks = requests.get( 'https://api.github.com/meta').json()['hooks'] # Check if the POST request is from github.com or GHE for block in hook_blocks: if ipaddress.ip_address(request_ip) in ipaddress.ip_network( block): break # the remote_addr is within the network range of github. else: if str(request_ip) != '127.0.0.1': abort(403) if request.headers.get('X-GitHub-Event') == "ping": return json.dumps({'msg': 'Hi!'}) if request.headers.get('X-GitHub-Event') != "push": return json.dumps({'msg': "wrong event type"}) repos = json.loads(io.open(REPOS_JSON_PATH, 'r').read()) payload = json.loads(request.data) repo_meta = { 'name': payload['repository']['name'], 'owner': payload['repository']['owner']['name'], } # Try to match on branch as configured in repos.json match = re.match(r"refs/heads/(?P<branch>.*)", payload['ref']) if match: repo_meta['branch'] = match.groupdict()['branch'] repo = repos.get( '{owner}/{name}/branch:{branch}'.format(**repo_meta), None) # Fallback to plain owner/name lookup if not repo: repo = repos.get('{owner}/{name}'.format(**repo_meta), None) if repo and repo.get('path', None): # Check if POST request signature is valid key = repo.get('key', None) if key: signature = request.headers.get('X-Hub-Signature').split( '=')[1] if type(key) == unicode: key = key.encode() mac = hmac.new(key, msg=request.data, digestmod=sha1) if not compare_digest(mac.hexdigest(), signature): abort(403) if repo.get('action', None): for action in repo['action']: subp = subprocess.Popen(action, cwd=repo.get('path', '.')) subp.wait() if repo.get('s3bucket', None): s3bucketname = repo.get('s3bucket') else: print('missing s3 bucketname') abort(500) if repo.get('s3key', None): s3key = repo.get('s3key') else: print('missing s3 filename') abort(500) print('s3 connection') if os.environ.get('USE_EC2', None) == 'true': provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() session = boto3.Session(aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, aws_session_token=creds.token) s3 = session.resource('s3').Bucket(s3bucketname) else: s3 = boto3.resource('s3') bucket = s3.Bucket(s3bucketname) json.load_s3 = lambda f: json.load(bucket.Object(key=f).get()['Body']) json.dump_s3 = lambda obj, f: bucket.Object(key=f).put(Body=json.dumps( obj)) #s3 fetch s3data = json.load_s3(s3key) datad = FilehashMap(s3data) commithash = payload['after'] for commit in payload['commits']: for z in commit['added']: print(z) datad.additem(z, commithash) for z in commit['modified']: print(z) datad.additem(z, commithash) for z in commit['removed']: datad.delitem(z) print(z) print('s3 upload') json.dump_s3(datad.displayhashmap(), s3key) #set perms s3objacl = s3.ObjectAcl(s3bucketname, s3key) response = s3objacl.put(ACL='public-read') print('s3 done') return 'OK'
def getCredentials(self): provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() access_key = creds.access_key return(access_key)
def get_upload_params(request): content_type = request.POST['type'] filename = request.POST['name'] dest = get_s3direct_destinations().get(request.POST['dest']) if not dest: data = json.dumps({'error': 'File destination does not exist.'}) return HttpResponse(data, content_type="application/json", status=400) key = dest.get('key') auth = dest.get('auth') allowed = dest.get('allowed') acl = dest.get('acl') bucket = dest.get('bucket') cache_control = dest.get('cache_control') content_disposition = dest.get('content_disposition') content_length_range = dest.get('content_length_range') server_side_encryption = dest.get('server_side_encryption') if not acl: acl = 'public-read' if not key: data = json.dumps({'error': 'Missing destination path.'}) return HttpResponse(data, content_type="application/json", status=403) if auth and not auth(request.user): data = json.dumps({'error': 'Permission denied.'}) return HttpResponse(data, content_type="application/json", status=403) if (allowed and content_type not in allowed) and allowed != '*': data = json.dumps({'error': 'Invalid file type (%s).' % content_type}) return HttpResponse(data, content_type="application/json", status=400) if hasattr(key, '__call__'): try: key = key(filename) except ValueError as ve: data = json.dumps({'error': 'Filename Invalid. %s' % ve}) return HttpResponse(data, content_type="application/json", status=400) elif key == '/': key = '${filename}' else: # The literal string '${filename}' is an S3 field variable for key. # https://aws.amazon.com/articles/1434#aws-table key = '%s/${filename}' % key access_key = getattr(settings, 'AWS_ACCESS_KEY_ID', None) secret_access_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None) token = None if access_key is None or secret_access_key is None: # Get credentials from instance profile if not defined in settings -- # this avoids the need to put access credentials in the settings.py file. # Assumes we're running on EC2. try: from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher except ImportError: InstanceMetadataProvider = None InstanceMetadataFetcher = None if all([InstanceMetadataProvider, InstanceMetadataFetcher]): provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() access_key = creds.access_key secret_access_key = creds.secret_key token = creds.token else: data = json.dumps({'error': 'Failed to access EC2 instance metadata due to missing dependency.'}) return HttpResponse(data, content_type="application/json", status=500) data = create_upload_data( content_type, key, acl, bucket, cache_control, content_disposition, content_length_range, server_side_encryption, access_key, secret_access_key, token ) return HttpResponse(json.dumps(data), content_type="application/json")
def __setupSparkSession__( self, jobConf: dict, ) -> SparkSession: ''' Init the Spark environemnt with few default configurations and start the spark session. ''' conf = SparkConf() # #Setup Spark Specific configurations # hmConf = { "spark.executor.pyspark.memory": "512m", "spark.debug.maxToStringFields": "5000", "spark.rps.askTimeout": "1200", "spark.network.timeout": "1200", "spark.maxRemoteBlockSizeFetchToMem": "512m", "spark.broadcast.blockSize": "16m", "spark.broadcast.compress": "true", "spark.rdd.compress": "true", "spark.io.compression.codec": "org.apache.spark.io.SnappyCompressionCodec", "spark.kryo.unsafe": "true", "spark.serializer": "org.apache.spark.serializer.KryoSerializer", "spark.kryoserializer.buffer": "10240", "spark.kryoserializer.buffer.max": "2040m", "hive.exec.dynamic.partition": "true", "hive.exec.dynamic.partition.mode": "nonstrict", "hive.warehouse.data.skiptrash": "true", "spark.sql.hive.metastorePartitionPruning": "true", "spark.sql.broadcastTimeout": "1200", "spark.sql.sources.partitionOverwriteMode": "dynamic", "spark.sql.orc.filterPushdown": "true", "spark.sql.orc.splits.include.file.footer": "true", "spark.sql.orc.cache.stripe.details.size": "1000", "spark.hadoop.parquet.enable.summary-metadata": "false", "spark.sql.parquet.mergeSchema": "false", "spark.sql.parquet.filterPushdown": "true", "spark.sql.parquet.fs.optimized.committer.optimization-enabled": "true", "spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version": "2", "spark.hadoop.mapreduce.fileoutputcommitter.cleanup-failures.ignored": "true" } for (k, v) in jobConf['sparkconfs'].items(): hmConf.set(k, v) conf.setAll(hmConf) # #Setup Hadoop Specific configurations # hdpCnf = SparkContext._jsc.hadoopConfiguration() hdpCnf.set('io.file.buffer.size', '65536') hdpCnf.set('mapreduce.fileoutputcommitter.algorithm.version', '2') for (k, v) in jobConf['hadoopconfs'].items(): hdpCnf.set(k, v) # #Setup AWS Specific configurations # if jobConf['appconfs']['runenv'].toUpperCase() == 'AWS': SparkContext.setSystemProperty( 'com.amazonaws.services.s3.enableV4', 'true') SparkContext.setSystemProperty( 'com.amazonaws.services.s3.enforceV4', 'true') conf.set( "spark.sql.parquet.output.committer.class", "com.amazon.emr.committer.EmrOptimizedSparkSqlParquetOutputCommitter" ) cred = None try: from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() hdpCnf.setAll({ 'fs.s3a.access.key': creds.access_key, 'fs.s3a.access.key': creds.secret_key, }) except: pass hdpCnf.setAll({ 'fs.s3a.server-side-encryption-algorithm': 'SSE-KMS', 'fs.s3.enableServerSideEncryption': 'true', 'fs.s3.enableServerSideEncryption': 'true', 'fs.s3.impl': 'org.apache.hadoop.fs.s3a.S3AFileSystem', 'fs.s3a.impl': 'org.apache.hadoop.fs.s3a.S3AFileSystem', 'fs.s3a.endpoint': "s3.%s.amazonaws.com" % (jobConf['appconfs']['appdefaults'] or 'us-east-1') }) spark = SparkSession \ .builder \ .config(conf=conf) \ .appName(jobConf['name'] or 'PySparkApp') \ .enableHiveSupport() \ .getOrCreate() sc = spark.sparkContext sc.setLogLevel(jobConf['appconfs']['logging']['sparkloglevel'] or 'INFO') if jobConf['appconfs']['logging']['sparkloglevel'] or 'INFO' == "DEBUG": msg = "" for k in sc._conf.getAll(): msg += "\t%50s -> %s\n" % (k[0], k[1]) log.debug( "Initiated SparkSesion with below confs,\n{}".format(msg)) return spark
import logging import paste.translogger import requests import os import boto3 from botocore.credentials import InstanceMetadataProvider, InstanceMetadataFetcher from requests_aws4auth import AWS4Auth secret_key = os.environ.get("SECRET_KEY") access_key = os.environ.get("ACCESS_KEY") if secret_key == None: logger = logging.getLogger("elasticsearch-service") logger.info("No params so attempt get config from machine") provider = InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) credentials = provider.load() access_key = credentials.access_key secret_key = credentials.secret_key region = os.environ.get('REGION') if region == None: region = "eu-central-1" def executeSignedPost(url, body): service = 'es' awsauth = AWS4Auth(access_key, secret_key, region, service) r = requests.post(url, auth=awsauth, json=body) result = r.json() return result
def get_instance_cred(): provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load().get_frozen_credentials() return creds