def get_user_data_configuration(): """Retrieve and update the application configuration with information from the user-data Returns: `None` """ from cloud_inquisitor import get_local_aws_session, app_config kms_region = app_config.kms_region session = get_local_aws_session() if session.get_credentials().method == 'iam-role': kms = session.client('kms', region_name=kms_region) else: sts = session.client('sts') audit_role = sts.assume_role(RoleArn=app_config.aws_api.instance_role_arn, RoleSessionName='cloud_inquisitor') kms = boto3.session.Session( audit_role['Credentials']['AccessKeyId'], audit_role['Credentials']['SecretAccessKey'], audit_role['Credentials']['SessionToken'], ).client('kms', region_name=kms_region) user_data_url = app_config.user_data_url res = requests.get(user_data_url) if res.status_code == 200: data = kms.decrypt(CiphertextBlob=b64decode(res.content)) kms_config = json.loads(zlib.decompress(data['Plaintext']).decode('utf-8')) app_config.database_uri = kms_config['db_uri'] else: raise RuntimeError('Failed loading user-data, cannot continue: {}: {}'.format(res.status_code, res.content))
def copy_objects_by_thread(thread_id, copied_objects): # HOWTO: each thread should have its own session # http://boto3.readthedocs.io/en/latest/guide/resources.html#multithreading session = boto3.session.Session() if args.optimal: # HOWTO: low-level control # http://boto3.readthedocs.io/en/latest/_modules/boto3/s3/transfer.html client_config = botocore.config.Config( max_pool_connections=args.max_concurrency) transfer_config = boto3.s3.transfer.TransferConfig( multipart_threshold=8 * 1024 * 1024, multipart_chunksize=8 * 1024 * 1024, max_concurrency=args.max_concurrency, num_download_attempts=5, max_io_queue=100, io_chunksize=256 * 1024) client = session.client('s3', config=client_config) else: s3 = session.resource('s3') client = session.client('s3') count = 0 while True: prefix = tasks.get() # HOWTO: list objects response = s3_client.list_objects_v2( Bucket=src_bucket.name, Prefix=prefix) # Important: using prefix to limit listing for content in response['Contents']: key = content['Key'] trace('thread %d copy object: s3://%s/%s' % \ (thread_id, src_bucket.name, key)) if not args.dryrun: if args.optimal: client.copy(CopySource={ 'Bucket': src_bucket.name, 'Key': key }, Bucket=dst_bucket.name, Key=key, Config=transfer_config) else: obj = s3.Object(dst_bucket.name, key) obj.copy_from(CopySource={ 'Bucket': src_bucket.name, 'Key': key }, ) count += 1 copied_objects[thread_id] = count tasks.task_done()
def get_aws_session(account): """Function to return a boto3 Session based on the account passed in the first argument. Args: account (:obj:`Account`): Account to create the session object for Returns: :obj:`boto3:boto3.session.Session` """ from cloud_inquisitor.config import dbconfig from cloud_inquisitor.plugins.types.accounts import AWSAccount if not isinstance(account, AWSAccount): raise InquisitorError('Non AWSAccount passed to get_aws_session, got {}'.format(account.__class__.__name__)) # If no keys are on supplied for the account, use sts.assume_role instead session = get_local_aws_session() if session.get_credentials().method == 'iam-role': sts = session.client('sts') else: # If we are not running on an EC2 instance, assume the instance role # first, then assume the remote role temp_sts = session.client('sts') audit_sts_role = temp_sts.assume_role( RoleArn=app_config.aws_api.instance_role_arn, RoleSessionName='inquisitor' ) sts = boto3.session.Session( audit_sts_role['Credentials']['AccessKeyId'], audit_sts_role['Credentials']['SecretAccessKey'], audit_sts_role['Credentials']['SessionToken'] ).client('sts') role = sts.assume_role( RoleArn='arn:aws:iam::{}:role/{}'.format( account.account_number, dbconfig.get('role_name', default='cinq_role') ), RoleSessionName='inquisitor' ) sess = boto3.session.Session( role['Credentials']['AccessKeyId'], role['Credentials']['SecretAccessKey'], role['Credentials']['SessionToken'] ) return sess
def plugin_write(vl, config): try: session = boto3.session.Session(region_name=config.aws_region) client_config = botocore.client.Config(connect_timeout=5, read_timeout=5) client = session.client('cloudwatch', config=client_config) metrics_list = list(metrics(vl, config)) ts = datetime.fromtimestamp(vl.time) data = [] for i, v in enumerate(vl.values): fullname, unit, dims = metrics_list[i] name = fullname[:255] if len(name) < len(fullname): collectd.warning( 'Metric name was truncated for CloudWatch: {}'.format( fullname)) data.append( dict(MetricName=name, Timestamp=ts, Value=v, Unit=unit, Dimensions=dims)) client.put_metric_data(Namespace=vl.plugin, MetricData=data) except Exception, e: collectd.error(str(e))
def set_client(): client = session.client(service_name=SERVICE_NAME, region_name=REGION_NAME, endpoint_url=ENDPOINT_URL, aws_access_key_id=ACCESS_ID, aws_secret_access_key=SECRET_KEY) return client
def create_client_with_profile(profile_name, region, resource_name='ec2'): """ Create a new boto3 client with a boto3 profile in ~/.aws/credentials Args: profile_name (str): The name of the profile that you have set in your ~/.aws/credentials profile. region (str): The aws region you want to connect to. resource_name (str): Valid aws resource. default=ec2 Basic Usage: >>> client, err_msg = create_client_with_profile('lab01', 'us-west-2') Returns: Tuple (botocore.client.EC2, str) """ client = None err_msg = '' try: session = (boto3.session.Session(profile_name=profile_name, region_name=region)) client = session.client(resource_name) except Exception as e: err_msg = str(e) return client, err_msg
def demo(): region_id = region_select() session = boto3.session.Session() client = session.client('ec2', region_name=region_id) ec2 = session.resource('ec2', region_name=region_id) ec2operat = EC2_Operate(client, ec2, region_id) print ec2operat.get_availbility_zone()
def plugin_write(vl, config): try: session = boto3.session.Session(region_name=config.aws_region) client_config = botocore.client.Config(connect_timeout=5, read_timeout=5) client = session.client('cloudwatch', config=client_config) metrics_list = list(metrics(vl, config)) ts = datetime.fromtimestamp(vl.time) data = [] for i, v in enumerate(vl.values): fullname, unit, dims = metrics_list[i] name = fullname[:255] if len(name) < len(fullname): collectd.warning('Metric name was truncated for CloudWatch: {}'.format(fullname)) data.append(dict( MetricName=name, Timestamp=ts, Value=v, Unit=unit, Dimensions=dims )) client.put_metric_data(Namespace=vl.plugin, MetricData=data) except Exception, e: collectd.error(str(e))
def deploy_iam_roles(session): role_dir = 'iam/roles' # get list of roles in iam dir iam_role_files = os.listdir(role_dir) # create session iam_client = session.client('iam') for role_file in os.listdir(role_dir): role_name, _ = os.path.splitext(role_file) # load in role and policy configs role = json.load(open('/'.join((role_dir, role_file)))) policy = json.load(open('iam/policies/{}.json'.format(role['policy']))) print 'creating role: {}'.format(role_name) try: iam_client.create_role( RoleName=role_name, AssumeRolePolicyDocument=json.dumps(policy)) except: print '\trole already exists'
def get_secret(secret_name: str) -> Optional[str]: """ Retrieve secret with name 'secret_name' from AWS Secrets Manager. :param secret_name: Name of the secret to retrieve. :return: The secret. """ AWS_REGION = current_app.config["AWS_REGION"] # Create a session with the Secrets Manager session = boto3.session.Session() client = session.client(service_name="secretsmanager", region_name=AWS_REGION) # Try to retrieve the secret from AWS Secrets Manager try: secret_value = client.get_secret_value(SecretId=secret_name) except ClientError as e: logging.error(f"AWS error: {e}. (Region={AWS_REGION}).") return None else: # Decrypts secret using the associated KMS CMK. if "SecretString" in secret_value: return json.loads(secret_value["SecretString"])[secret_name] else: return base64.b64decode(secret_value["SecretBinary"])
def ReadSecrets(self): # Create a Secrets Manager client session = None if self.profilename: # Use the specified profile session = boto3.session.Session(profile_name=self.profilename) else: # Pull the profile from the environment session = boto3.session.Session() client = session.client(service_name='secretsmanager', region_name=self.region) # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html # We rethrow the exception by default. try: response = client.get_secret_value(SecretId=self.secretname) jsondata = response['SecretString'] data = json.loads(jsondata) self.property_certfilebin = base64.b64decode(data['cert']) self.property_privatekeyfilebin = base64.b64decode(data['key']) except ClientError as e: self.logging.error( f"Get secrets failed: {e.response['Error']['Code']}") raise e
def list_files(bucket: str, prefix: str = "", profile_name: Optional[str] = None) -> List[str]: """ List up to 1000 files in a bucket. Parameters ---------- bucket : str prefix : str profile_name : str, optional AWS profile Returns ------- s3_paths : List[str] """ session = boto3.session.Session(profile_name=profile_name) conn = session.client("s3") keys = [] ret = conn.list_objects_v2(Bucket=bucket, Prefix=prefix) if "Contents" not in ret: return [] # Make this a generator in future and use the marker: # https://boto3.readthedocs.io/en/latest/reference/services/ # s3.html#S3.Client.list_objects for key in conn.list_objects_v2(Bucket=bucket, Prefix=prefix)["Contents"]: keys.append("s3://" + bucket + "/" + key["Key"]) return keys
def all_regions(args): session = boto3.session.Session( aws_access_key_id=args.access_key_id, aws_secret_access_key=args.secret_access_key) ec2 = session.client('ec2', region_name='us-west-2') regions = ec2.describe_regions() return [r['RegionName'] for r in regions['Regions']]
def __init__(self, service_name, aws_access_key_id, aws_secret_access_key, region_name): session = boto3.session.Session( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name) self.aws_client = session.client(service_name)
def create_presigned_post(bucket_name, object_name, fields=None, conditions=None, expiration=3600): """Generate a presigned URL S3 POST request to upload a file :param bucket_name: string :param object_name: string :param fields: Dictionary of prefilled form fields :param conditions: List of conditions to include in the policy :param expiration: Time in seconds for the presigned URL to remain valid :return: Dictionary with the following keys: url: URL to post to fields: Dictionary of form fields and values to submit with the POST :return: None if error. """ # Generate a presigned S3 POST URL session = boto3.session.Session(profile_name=AWS_PROFILE) s3_client = session.client('s3') try: response = s3_client.generate_presigned_post(bucket_name, object_name, Fields=fields, Conditions=conditions, ExpiresIn=expiration) except ClientError as e: logging.error(e) return None # The response contains the presigned URL and required fields return response
def get_rds_status(): session = boto3.Session( profile_name='--profile name goes here--') # aws credentials rds = session.client('rds') if now >= night_begin and now <= night_end: try: dbs = rds.describe_db_instances( DBInstanceIdentifier='temp-min' ) # checking RDS instance named temp-min if dbs['DBInstances'][0]['DBInstanceStatus'] == 'available': rds_status = 'The RDS temp instance is RUNNING' rds_color = green else: rds_status = 'The RDS temp instance is ' + dbs['DBInstances'][ 0]['DBInstanceStatus'] rds_color = orange except Exception as error: rds_status = 'The RDS temp instance does not exist' rds_color = red else: try: dbs = rds.describe_db_instances(DBInstanceIdentifier='temp-min') if dbs['DBInstances'][0]['DBInstanceStatus'] == 'available': rds_status = 'The RDS temp instance is RUNNING' rds_color = red else: rds_status = 'The RDS temp instance is ' + dbs['DBInstances'][ 0]['DBInstanceStatus'] rds_color = orange except Exception as error: rds_status = 'The RDS temp instance does not exist' rds_color = green return rds_status, rds_color
def test_transfer_methods_injected_to_client(self): session = boto3.session.Session(region_name='us-west-2') client = session.client('s3') self.assertTrue(hasattr(client, 'upload_file'), 'upload_file was not injected onto S3 client') self.assertTrue(hasattr(client, 'download_file'), 'download_file was not injected onto S3 client')
def s3_read(source: str, profile_name: Optional[str] = None) -> bytes: """ Read a file from an S3 source. Parameters ---------- source : str Path starting with s3://, e.g. 's3://bucket-name/key/foo.bar' profile_name : str, optional AWS profile Returns ------- content : bytes Raises ------ botocore.exceptions.NoCredentialsError Botocore is not able to find your credentials. Either specify profile_name or add the environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN. See https://boto3.readthedocs.io/en/latest/guide/configuration.html """ session = boto3.session.Session(profile_name=profile_name) s3 = session.client("s3") bucket_name, key = _s3_path_split(source) s3_object = s3.get_object(Bucket=bucket_name, Key=key) body = s3_object["Body"] return body.read()
def aws_session(aws_account=None, aws_role=None): if ArgumentParser.args.aws_role_override: aws_role = ArgumentParser.args.aws_role if aws_role and aws_account: role_arn = f"arn:aws:iam::{aws_account}:role/{aws_role}" session = boto3.session.Session( aws_access_key_id=ArgumentParser.args.aws_access_key_id, aws_secret_access_key=ArgumentParser.args.aws_secret_access_key, region_name="us-east-1", ) sts = session.client("sts") token = sts.assume_role( RoleArn=role_arn, RoleSessionName=f"{aws_account}-{str(uuid.uuid4())}") credentials = token["Credentials"] return boto3.session.Session( aws_access_key_id=credentials["AccessKeyId"], aws_secret_access_key=credentials["SecretAccessKey"], aws_session_token=credentials["SessionToken"], ) else: return boto3.session.Session( aws_access_key_id=ArgumentParser.args.aws_access_key_id, aws_secret_access_key=ArgumentParser.args.aws_secret_access_key, )
def do_auth_command(self, bot, connection, event, command, parameters, reply_target, auth_level): if command not in self.auth_commands: return False # not for us if command == 'backup': creds = bot.db.get('s3_credentials') # <access key>|<secret key>|<bucket name>|<uploaded file name> akey, skey, bname, uname = creds.split('|') session = boto3.session.Session( aws_access_key_id = akey, aws_secret_access_key = skey, ) client = session.client('s3') transfer = boto3.s3.transfer.S3Transfer(client) transfer.upload_file( bot.module_parameters['database:name'], bname, uname, extra_args = { 'StorageClass': 'STANDARD_IA', }, ) bot.send(connection, reply_target, bot.db.get_random('yes'), event) return True return False
def get_org_accounts(filter_current_account=False): session = aws_session() client = session.client('organizations') accounts = [] try: response = client.list_accounts() accounts = response.get('Accounts', []) while response.get('NextToken') is not None: response = client.list_accounts(NextToken=response['NextToken']) accounts.extend(response.get('Accounts', [])) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'AccessDeniedException': log.error( 'AWS error - missing permissions to list organization accounts' ) else: raise filter_account_id = current_account_id() if filter_current_account else -1 accounts = [ aws_account['Id'] for aws_account in accounts if aws_account['Id'] != filter_account_id ] for account in accounts: log.debug('AWS found org account {}'.format(account)) log.info('AWS found a total of {} org accounts'.format(len(accounts))) return accounts
def main(): from boto3 import session, ec2 from pprint import PrettyPrinter utc = UTC() pp = PrettyPrinter(indent=4) args = get_args() delta_kill = datetime.now(utc) - timedelta(hours=args.terminate) delta_warn = datetime.now(utc) - timedelta(hours=args.warn) session = session.Session(region_name=args.region, profile_name=args.profile) ec2client = session.client('ec2') response = ec2client.describe_instances() warn_instances = [] for reservation in response['Reservations']: for instance in reservation['Instances']: launchtime = instance['LaunchTime'] if launchtime < delta_kill and args.yes: print("Terminating instance", instance['InstanceId']) # ec2client.terminate_instances(InstanceIds=[instance[u'InstanceId']]) elif launchtime < delta_kill and not args.yes: print("Skipping instance", instance['InstanceId']) elif launchtime < delta_warn: warn_instances += instance if warn_instances: print("The following instances are more than ", args.warn, "hrs old.") pp.pprint(warn_instances)
def create_s3_account(sender, instance, created, **kwargs): if created: import requests import json url = settings.BACKEND_ENDPOINT_URL + '/riak-cs/user' data = {'email': instance.email, 'name': instance.get_full_name()} response = requests.post(url, json=data) response_dict = json.loads(response.text) S3Account.objects.create(user=instance, id=response_dict['id'], key_id=response_dict['key_id'], key_secret=response_dict['key_secret'], status=response_dict['status']) session = boto3.session.Session() s3client = session.client( 's3', use_ssl=False, endpoint_url=settings.BACKEND_ENDPOINT_URL, aws_access_key_id=response_dict['key_id'], aws_secret_access_key=response_dict['key_secret']) try: s3client.create_bucket(Bucket=str(uuid.uuid4())) except ClientError as e: print(str(e)) pass
def download_aws(self): #bucket_name = frappe.db.get_value("Aws Setup", None, "s3_bucket") #report_name = frappe.db.get_value("Aws Setup", None, "report_name") bucket_name = frappe.db.get_value("Monthly Recurring Setup", None, "s3_bucket") report_name = frappe.db.get_value("Monthly Recurring Setup", None, "report_name") session = boto3.session.Session(region_name='us-east-1') s3client = session.client('s3', config= boto3.session.Config(signature_version='s3v4')) #resp = s3client.list_objects(Bucket=bucket_name) # print names of all objects first_day_of_this_month = frappe.utils.data.get_first_day (self.transaction_date, 0, 0) next_month = frappe.utils.data.get_first_day (self.transaction_date, 0, 1) #for obj in resp['Contents']: # if "Billing-aptech-Manifest.json" in str(obj['Key']): # frappe.msgprint("Object Name: " + obj['Key']) # Get the service client s3 = boto3.client('s3') # Download object at bucket-name with key-name to tmp.txt path = "/home/ubuntu/frappe-bench/apps/aptitudetech_private/aptitudetech_private/aptitudetech_private/doctype/aws_transaction/" s3client.download_file(bucket_name, report_name + "/Billing-aptech/" + str(first_day_of_this_month).replace("-", "") + "-" + str(next_month).replace("-","") + "/Billing-aptech-Manifest.json", path + "tmp1.json") path = "/home/ubuntu/frappe-bench/apps/aptitudetech_private/aptitudetech_private/aptitudetech_private/doctype/aws_transaction/" j = json.loads(open(path + "tmp1.json", 'r').read()) s3client.download_file(bucket_name, str(j["reportKeys"]).replace("[u'","").replace("']",""), path + "tmp2.zip") zip_ref = zipfile.ZipFile(path + "tmp2.zip", 'r') zip_ref.extractall(path) zip_ref.close()
def aws_sts_role(self, api_call, role_arn): try: # experimenting the aws STS roles configurations session = boto3.session.Session(profile_name="betaDev", region_name=self.region) # Session = boto3.Session(region_name=self.REGION) client = session.client(api_call) logger.debug(client) assume_role = client.assume_role( RoleArn=role_arn, RoleSessionName='AssumeRoleSession', PolicyArns=[ { "arn": "arn:aws:iam::938819073054:policy/awsCustomPolicy" }, ], # ExternalId= 'ec2filter' ) logger.info(assume_role) except Exception as end_point_err: logger.exception("Logging Exception: " + str(end_point_err) + "\n") # raise sys.exit(1) else: return assume_role
def process_cf_file(self, args): try: cf_params_local = copy.deepcopy(args[0]) cluster = args[1] elb_name_suffix = args[2] env = args[3] filename = args[4] has_ecs_service = args[5] listener_port = args[6] region = args[7] session = boto3.session.Session() if has_ecs_service: elb_name = 'ecs-elb-' + cluster if elb_name_suffix is not None: elb_name = "-".join([elb_name, elb_name_suffix]) self.populate_ecs_service_params(session, cf_params_local, cluster, elb_name, env, region, listener_port) # Skip non-cf files ext = filename.split('.')[-1] if ext != 'template' and ext != 'yml': return cf_client = session.client('cloudformation', region_name=region) name = filename.split('/')[-1].split('.')[0] logging.info("%s: Processing CloudFormation Template" % filename) cf_params_local['name'] = name parameters = [{'ParameterKey': 'name', 'ParameterValue': name}] if name is None or name in filename: with open(filename, 'r') as f_h: try: cf_template = f_h.read() except: logging.exception("%s: Error reading file." % (filename)) self.catfile(filename) raise validate_response = self.validate_template(cf_client, cf_template, filename) service_name = "%s-%s-%s" % (env, name, cluster) if elb_name_suffix is not None: service_name = "-".join([service_name, elb_name_suffix]) cf_command = cf_client.create_stack existing_stack_id = self.find_existing_stack(cf_client, cf_params_local, service_name) if existing_stack_id is not None: cf_command = cf_client.update_stack self.populate_cf_params(cf_params_local, existing_stack_id, filename, parameters, validate_response) logging.info("%s: Updating CloudFormation Stack" % (service_name)) try: cf_response = cf_command(StackName=service_name, TemplateBody=cf_template, Parameters=parameters, Capabilities=["CAPABILITY_IAM"]) creating_stack_id = cf_response['StackId'] stack_status = self.wait_for_stack_creation(cf_client, creating_stack_id, service_name) except botocore.exceptions.ClientError as e: if e.response["Error"]["Message"] == 'No updates are to be performed.': logging.info("%s: No updates to be performed, CF update succeeded." % service_name) else: raise self.q.put("%s Succeeded" % filename) logging.info("%s Succeeded" % filename) except Exception as e: logging.error("%s: Error executing CloudFormation Stack" % filename) logging.exception(e) self.q.put("%s Failed" % filename)
def check(session): client = session.client('s3') try: response = client.list_buckets() except Exception as e: logging.exception(e) return None else: return [b['Name'] for b in response['Buckets']]
async def start_up(app, loop): app.cache = {} session = boto3.session.Session(region_name=os.environ['AWS_REGION']) app.client = session.client('s3') app.redis = await aioredis.create_pool(address=(os.environ['REDIS_HOST'], os.environ['REDIS_PORT']), minsize=5, maxsize=10, loop=loop)
def __init__(self, aws_access_key, aws_secret_key, bucket_region): self.aws_access_key = aws_access_key self.aws_secret_key = aws_secret_key session = boto3.session.Session(region_name=bucket_region) self.s3_obj = session.client( 's3', aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key, config=boto3.session.Config(signature_version='s3v4'))
def __send_ses_email(sender, recipients, subject, html_body, text_body): """Send an email using SES Args: sender (str): From email address recipients (`1ist` of `str`): List of recipient email addresses subject (str): Subject of the email html_body (str): HTML body of the email text_body (str): Text body of the email Returns: `None` """ access_key = app.config.get('AWS_API_ACCESS_KEY') secret_key = app.config.get('AWS_API_SECRET_KEY') source_arn = dbconfig.get('source_arn', NS_EMAIL) return_arn = dbconfig.get('return_path_arn', NS_EMAIL) if access_key and secret_key: session = boto3.session.Session(access_key, secret_key) else: session = boto3.session.Session() ses = session.client('ses', region_name=dbconfig.get('ses_region', NS_EMAIL, 'us-west-2')) body = {} if html_body: body['Html'] = { 'Data': html_body } if text_body: body['Text'] = { 'Data': text_body } ses_options = { 'Source': sender, 'Destination': { 'ToAddresses': recipients }, 'Message': { 'Subject': { 'Data': subject }, 'Body': body } } # Set SES options if needed if source_arn and return_arn: ses_options.update({ 'SourceArn': source_arn, 'ReturnPathArn': return_arn }) ses.send_email(**ses_options)
def get_latest_ubuntu_ami() -> str: session = boto3.session.Session() images = session.client('ec2').describe_images(Filters=[{ 'Name': 'name', 'Values': ['ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server*'] }]) return sorted(images['Images'], key=lambda i: i['CreationDate'])[-1]['ImageId']
def create_instance(region, instance_name, centos_version, instancetype, vpc, publicaccess, azid): session = boto3.session.Session() client = session.client('ec2', region_name=region) ec2 = session.resource('ec2', region_name=region) ec2operat = EC2_Operate(client, ec2, region) instancetag = ec2operat.ec2_create(instance_name, centos_version, instancetype, vpc, publicaccess, azid) #import to db get_instance_info_bytags(region, [instancetag]) return instancetag
def getResourceFromSession(resourceName, region): credentials = getCredentials() session = boto3.session.Session() resource = session.client( resourceName, region, aws_access_key_id=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_session_token=credentials['SessionToken']) return resource
def get_buckets(self): session = boto3.session.Session() s3client = session.client('s3', use_ssl=False, endpoint_url=settings.BACKEND_ENDPOINT_URL, aws_access_key_id=self.key_id, aws_secret_access_key=self.key_secret) response = s3client.list_buckets() return [bucket['Name'] for bucket in response['Buckets']]
def populate_ecs_service_params(self, session, cf_params, cluster, elb_name, env, region, listener_port): elb_client = session.client('elbv2', region_name=region) balancer_arn, vpc_id = ApplyECS.get_load_balancer(elb_client, elb_name, cluster, env) listener_arn = ApplyECS.get_elb_listener(elb_client, balancer_arn, port=listener_port) cf_params['vpcid'] = vpc_id cf_params['listenerarn'] = listener_arn response = elb_client.describe_rules(ListenerArn=listener_arn) rules = response['Rules'] existing_priorities = set([rule['Priority'] for rule in rules]) if len(existing_priorities) >= 11: logging.error("Listener %s already has %d rules, cannot add more services" % (listener_arn, len(existing_priorities))) raise Exception("Listener %s already has %d rules, cannot add more services" % (listener_arn, len(existing_priorities))) for i in range(10, 21): if str(i) not in existing_priorities: cf_params['priority'] = str(i) break
def deploy_lambda_function(session, function_name): # load in configs function_cfg = json.load(open('lambda/conf/{}.json'.format(function_name))) default_cfg = json.load(open('lambda/conf/defaults.json')) default_cfg.update(function_cfg) cfg = default_cfg lambda_client = session.client('lambda') try: # function already exists, so update it print 'checking if function exists: {}'.format(function_name) lambda_client.get_function_configuration( FunctionName=function_name) print '\tupdating function configuration: {}'.format(function_name) lambda_client.update_function_configuration( FunctionName=function_name, Role=session.resource('iam').Role(cfg['role']).arn, Handler=cfg['handler'], Description=cfg.get('description', None), Timeout=cfg['timeout'], MemorySize=cfg['memory'], VpcConfig=cfg.get('vpc', {})) print '\tupdating function code: {}'.format(function_name) lambda_client.update_function_code( FunctionName=function_name, ZipFile=open('_deploy/_zip/{}.zip'.format(function_name), 'rb').read()) except: # function is new, so create it print '\tcreating new function: {}'.format(function_name) lambda_client.create_function( FunctionName=function_name, Runtime=cfg['runtime'], Role=session.resource('iam').Role(cfg['role']).arn, Code={ 'ZipFile': open('_deploy/_zip/{}.zip'.format(function_name), 'rb').read() }, Handler=cfg['handler'], Description=cfg.get('description', None), Timeout=cfg['timeout'], MemorySize=cfg['memory'], VpcConfig=cfg.get('vpc', {}))
def security_credentials_role_name(): role_arn = _get_role_arn() credentials = _credential_map.get(role_arn) # Refresh credentials if going to expire soon. now = datetime.datetime.now(tz=dateutil.tz.tzutc()) if not credentials or credentials['Expiration'] < now + _refresh_timeout: try: # Use any boto3 credential provider except the instance metadata provider. botocore_session = botocore.session.Session() botocore_session.get_component('credential_provider').remove('iam-role') session = boto3.session.Session(botocore_session=botocore_session) credentials = session.client('sts').assume_role(RoleArn=role_arn, RoleSessionName="ectou-metadata")['Credentials'] credentials['LastUpdated'] = now _credential_map[role_arn] = credentials except Exception as e: bottle.response.status = 404 bottle.response.content_type = 'text/plain' # EC2 serves json as text/plain return json.dumps({ 'Code': 'Failure', 'Message': e.message, }, indent=2) # Return current credential. bottle.response.content_type = 'text/plain' # EC2 serves json as text/plain return json.dumps({ 'Code': 'Success', 'LastUpdated': _format_iso(credentials['LastUpdated']), "Type": "AWS-HMAC", 'AccessKeyId': credentials['AccessKeyId'], 'SecretAccessKey': credentials['SecretAccessKey'], 'Token': credentials['SessionToken'], 'Expiration': _format_iso(credentials['Expiration']) }, indent=2)
def get_connection(service, module=None, region=None, key=None, keyid=None, profile=None): ''' Return a boto connection for the service. .. code-block:: python conn = __utils__['boto.get_connection']('ec2', profile='custom_profile') ''' module = module or service cxkey, region, key, keyid = _get_profile(service, region, key, keyid, profile) cxkey = cxkey + ':conn' if cxkey in __context__: return __context__[cxkey] try: session = boto3.session.Session(aws_access_key_id=keyid, aws_secret_access_key=key, region_name=region) if session is None: raise SaltInvocationError('Region "{0}" is not ' 'valid.'.format(region)) conn = session.client(module) if conn is None: raise SaltInvocationError('Region "{0}" is not ' 'valid.'.format(region)) except boto.exception.NoAuthHandlerFound: raise SaltInvocationError('No authentication credentials found when ' 'attempting to make boto {0} connection to ' 'region "{1}".'.format(service, region)) __context__[cxkey] = conn return conn
log_lvl = os.getenv('LOG_LVL', default='WARNING') if log_lvl == 'ERROR': logger.setLevel(logging.ERROR) elif log_lvl == 'INFO': logger.setLevel(logging.INFO) elif log_lvl == 'DEBUG': logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.WARNING) logger.addHandler(logging.StreamHandler()) session = boto3.session.Session(region_name=REGION) sqs = session.client('sqs') s3 = session.client('s3') def pull_files(message_URL): ''' pull message form message_URL ''' msgs = {} while 'Messages' not in msgs: logger.debug('pull message from SQS: {}'.format(message_URL)) while 'Messages' not in msgs: try: msgs = sqs.receive_message(QueueUrl=message_URL) except botocore.exceptions.ClientError as err: logger.debug(traceback.format_exc())
import boto3 import boto3.session from util import AESCipher session = boto3.session.Session(region_name='eu-west-1') s3Client = session.client('s3') data = open('manage.py', 'rb') cipher = AESCipher(key='1234567890123456') encrypted = cipher.encrypt(data.read().decode('utf-8')) print(encrypted) s3Client.put_object(Bucket='reciproco',Key='manage.py', Body=encrypted) print(s3Client.generate_presigned_url('get_object', Params = {'Bucket': 'reciproco', 'Key': 'manage.py'}, ExpiresIn = 100)) new_cipher = AESCipher(key='1234567890123456') response = s3Client.get_object(Bucket='reciproco',Key='manage.py') i = response["Body"].read() print(i) decrypted = new_cipher.decrypt(i) print(decrypted.decode('utf-8'))
def run_berry(args): try: with open(args.config_file) as fd: config = yaml.load(fd) except Exception as e: logging.warn('Could not load configuration from {}: {}'.format(args.config_file, e)) config = {} application_id = args.application_id or config.get('application_id') mint_bucket = args.mint_bucket or config.get('mint_bucket') local_directory = args.local_directory if not application_id: raise UsageError('Application ID missing, please set "application_id" in your configuration YAML') if not mint_bucket: raise UsageError('Mint Bucket is not configured, please set "mint_bucket" in your configuration YAML') if args.aws_credentials_file: aws_credentials = use_aws_credentials(application_id, args.aws_credentials_file) else: aws_credentials = {} session = boto3.session.Session(**aws_credentials) s3 = session.client('s3') while True: for fn in ['user', 'client']: key_name = '{}/{}.json'.format(application_id, fn) try: local_file = os.path.join(local_directory, '{}.json'.format(fn)) tmp_file = local_file + '.tmp' response = None retry = 3 while retry: try: response = s3.get_object(Bucket=mint_bucket, Key=key_name) retry = False except botocore.exceptions.ClientError as e: # more friendly error messages # https://github.com/zalando-stups/berry/issues/2 status_code = e.response.get('ResponseMetadata', {}).get('HTTPStatusCode') msg = e.response['Error'].get('Message') error_code = e.response['Error'].get('Code') endpoint = e.response['Error'].get('Endpoint', '') retry -= 1 if error_code == 'InvalidRequest' and 'Please use AWS4-HMAC-SHA256.' in msg: logging.debug(('Invalid Request while trying to read "{}" from mint S3 bucket "{}". ' + 'Retrying with signature version v4! ' + '(S3 error message: {})').format( key_name, mint_bucket, msg)) s3 = session.client('s3', config=Config(signature_version='s3v4')) elif error_code == 'PermanentRedirect' and endpoint.endswith('.amazonaws.com'): region = get_bucket_region(s3, mint_bucket, endpoint) logging.debug(('Got Redirect while trying to read "{}" from mint S3 bucket "{}". ' + 'Retrying with region {}, endpoint {}! ' + '(S3 error message: {})').format( key_name, mint_bucket, region, endpoint, msg)) s3 = session.client('s3', region) elif status_code == 403: logging.error(('Access denied while trying to read "{}" from mint S3 bucket "{}". ' + 'Check your IAM role/user policy to allow read access! ' + '(S3 error message: {})').format( key_name, mint_bucket, msg)) retry = False elif status_code == 404: logging.error(('Credentials file "{}" not found in mint S3 bucket "{}". ' + 'Mint either did not sync them yet or the mint configuration is wrong. ' + '(S3 error message: {})').format( key_name, mint_bucket, msg)) retry = False else: logging.error('Could not read from mint S3 bucket "{}": {}'.format( mint_bucket, e)) retry = False if response: body = response['Body'] json_data = body.read() # check that the file contains valid JSON new_data = json.loads(json_data.decode('utf-8')) try: with open(local_file, 'r') as fd: old_data = json.load(fd) except: old_data = None # check whether the file contents changed if new_data != old_data: with open(tmp_file, 'wb') as fd: fd.write(json_data) os.rename(tmp_file, local_file) logging.info('Rotated {} credentials for {}'.format(fn, application_id)) except: logging.exception('Failed to download {} credentials'.format(fn)) if args.once: break time.sleep(args.interval) # pragma: no cover