def connect_to_region(cls, region, session=None, access_key=None, secret_key=None, **kwargs): """ Connect to an AWS region. This method has been deprecated in favor of :meth:`~.connect` Parameters ---------- region : str Name of an AWS region session : :class:`~botocore.session.Session`, optional The Session object to use for the connection access_key : str, optional If session is None, set this access key when creating the session secret_key : str, optional If session is None, set this secret key when creating the session **kwargs : dict Keyword arguments to pass to the constructor """ if session is None: session = botocore.session.get_session() if access_key is not None: session.set_credentials(access_key, secret_key) service = session.get_service('dynamodb') return cls(service, service.get_endpoint(region), **kwargs)
def upload_to_sandbox(path): print('File \'%s\' uploading started...' % path.replace('api/test/../../', '')) start_time = str(int(time.time())) session = botocore.session.get_session() session.set_credentials(access_key="AKIAIA5IPHVWBGQSSBBA", secret_key="uhsuCK/U8hBWYkMHPoB9dGFJHxjTFK4y63cx3D/y") client = session.create_client('s3', region_name='us-west-2') f = open(path, 'rb') key = '/'.join(unicode(uuid4()).split('-') + [os.path.basename(path)]) response = client.put_object( Body=f.read(), Bucket='bkstg-sandbox', Key=key, ContentType=mimetypes.guess_type(path)[0] ) end_time = str(int(time.time())) print('File was uploaded in %s seconds.\n' % (int(end_time) - int(start_time))) return key # if __name__ == '__main__': # Upload.upload_to_sandbox(path=TestData.image_path)
def mk_boto_session( profile: Optional[str] = None, creds: Optional[ReadOnlyCredentials] = None, region_name: Optional[str] = None, ) -> Session: """Get botocore session with correct `region` configured :param profile: profile name to lookup :param creds: Override credentials with supplied data :param region_name: default region_name to use if not configured for a given profile """ session = botocore.session.Session(profile=profile) if creds is not None: session.set_credentials(creds.access_key, creds.secret_key, creds.token) _region = session.get_config_variable("region") if _region is None: if region_name is None or region_name == "auto": _region = auto_find_region(session, default="us-west-2") else: _region = region_name session.set_config_variable("region", _region) return session
def build_vpc(aws_creds, region, cidr=""): print('Building VPC for %s network' % str(cidr)) session = botocore.session.get_session() session.set_credentials(aws_creds.access_key, aws_creds.secret_key) ec2 = session.get_service('ec2') operation = ec2.get_operation('CreateVpc') endpoint = ec2.get_endpoint(region) http_response, response_data = operation.call(endpoint, CidrBlock=cidr) p3.pprint(str(http_response.status_code) + " - " + http_response.reason) p3.pprint(response_data) if http_response.status_code != 200: raise(ApiException) vpc_id = response_data['Vpc']['VpcId'] operation = ec2.get_operation('CreateTags') http_response, response_data = operation.call(endpoint, Resources=[vpc_id], Tags=[{"Key": "Name", "Value": 'Meetup' }], ) return vpc_id
def main(): session = botocore.session.Session() session.set_credentials(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY) ec2 = session.get_service('ec2') endpoint = ec2.get_endpoint(AWS_DEFAULT_REGION) op_din = ec2.get_operation('DescribeInstances') op_stop = ec2.get_operation('StopInstances') http_response, reservation_data = op_din.call(endpoint, filters=[{"Name":"instance-state-name","Values":["running"]}] ) if not http_response.ok: print(reservation_data['Errors']) sys.exit(2) else: reservations = reservation_data['Reservations'] #Pull the instances out of reservations instances = [] for reservation in reservations: instances += reservation['Instances'] print("Found %i instances in running state" % len(instances)) instance_ids = [instance['InstanceId'] for instance in instances] http_response, data = op_stop.call(endpoint, instance_ids = instance_ids) if not http_response.ok: print(data['Errors']) sys.exit(2) for instance in data['StoppingInstances']: print("Instance %s now stopping" % instance['InstanceId'])
def connect_to_host(cls, host='localhost', port=8000, is_secure=False, session=None, access_key=None, secret_key=None, **kwargs): """ Connect to a specific host. This method has been deprecated in favor of :meth:`~.connect` Parameters ---------- host : str, optional Address of the host (default 'localhost') port : int, optional Connect to the host on this port (default 8000) is_secure : bool, optional Enforce https connection (default False) session : :class:`~botocore.session.Session`, optional The Session object to use for the connection access_key : str, optional If session is None, set this access key when creating the session secret_key : str, optional If session is None, set this secret key when creating the session **kwargs : dict Keyword arguments to pass to the constructor """ if session is None: session = botocore.session.get_session() if access_key is not None: session.set_credentials(access_key, secret_key) url = "http://%s:%d" % (host, port) service = session.get_service('dynamodb') endpoint = service.get_endpoint( 'local', endpoint_url=url, is_secure=is_secure) return cls(service, endpoint, **kwargs)
def connect_to_region(cls, region, session=None, access_key=None, secret_key=None, **kwargs): """ Connect to an AWS region. This method has been deprecated in favor of :meth:`~.connect` Parameters ---------- region : str Name of an AWS region session : :class:`~botocore.session.Session`, optional The Session object to use for the connection access_key : str, optional If session is None, set this access key when creating the session secret_key : str, optional If session is None, set this secret key when creating the session **kwargs : dict Keyword arguments to pass to the constructor """ warnings.warn("connect_to_region is deprecated and will be removed. " "Use connect instead.") if session is None: session = botocore.session.get_session() if access_key is not None: session.set_credentials(access_key, secret_key) client = session.create_client('dynamodb', region) return cls(client, **kwargs)
def build_elb(aws_creds, region, lb_name="", subnets=[], elb_sg_id=""): print('Building elb') session = botocore.session.get_session() session.set_credentials(aws_creds.access_key, aws_creds.secret_key) elb = session.get_service('elb') operation = elb.get_operation('CreateLoadBalancer') endpoint = elb.get_endpoint(region) http_response, response_data = operation.call(endpoint, LoadBalancerName=lb_name, Listeners=[ {"Protocol": 'http', "LoadBalancerPort": 80, "InstanceProtocol": 'http', "InstancePort": 80}], Subnets=subnets, SecurityGroups=[elb_sg_id], Scheme='internal') p3.pprint(str(http_response.status_code) + " - " + http_response.reason) p3.pprint(response_data) if http_response.status_code != 200: raise(ApiException) elb_name = response_data['DNSName'] return elb_name
def boto3_session(profile_name=None, config_file=None, credentials_file=None, existing_creds=None): """ returns a boto3 session using cached credentials Allows for sharing credentials from cli commands with boto sessions """ # Construct botocore session with cache session = botocore.session.get_session() if profile_name is not None: session.set_config_variable("profile", profile_name) if config_file is not None: session.set_config_variable("config_file", config_file) if credentials_file is not None: session.set_config_variable("credentials_file", credentials_file) if existing_creds is not None: session.set_credentials( access_key=existing_creds["aws_access_key_id"], secret_key=existing_creds["aws_secret_access_key"], token=existing_creds["aws_session_token"], ) session.get_component("credential_provider").get_provider( "assume-role").cache = credentials.JSONFileCache(cli_cache) return boto3.Session(botocore_session=session)
def __init__(self, s3_staging_dir=None, access_key=None, secret_key=None, region_name=None, schema_name='default', profile_name=None, credential_file=None, jvm_path=None, jvm_options=None, converter=None, formatter=None, driver_path=None, **driver_kwargs): if s3_staging_dir: self.s3_staging_dir = s3_staging_dir else: self.s3_staging_dir = os.getenv(self._ENV_S3_STAGING_DIR, None) assert self.s3_staging_dir, 'Required argument `s3_staging_dir` not found.' assert schema_name, 'Required argument `schema_name` not found.' self.schema_name = schema_name if credential_file: self.access_key = None self.secret_key = None self.token = None self.credential_file = credential_file assert self.credential_file, 'Required argument `credential_file` not found.' self.region_name = region_name assert self.region_name, 'Required argument `region_name` not found.' else: import botocore.session session = botocore.session.get_session() if access_key and secret_key: session.set_credentials(access_key, secret_key) if profile_name: session.set_config_variable('profile', profile_name) if region_name: session.set_config_variable('region', region_name) credentials = session.get_credentials() self.access_key = credentials.access_key assert self.access_key, 'Required argument `access_key` not found.' self.secret_key = credentials.secret_key assert self.secret_key, 'Required argument `secret_key` not found.' self.token = credentials.token self.credential_file = None self.region_name = session.get_config_variable('region') assert self.region_name, 'Required argument `region_name` not found.' self._start_jvm(jvm_path, jvm_options, driver_path) props = self._build_driver_args(**driver_kwargs) jpype.JClass(ATHENA_DRIVER_CLASS_NAME) self._jdbc_conn = jpype.java.sql.DriverManager.getConnection( ATHENA_CONNECTION_STRING.format(region=self.region_name, schema=schema_name), props) self._converter = converter if converter else JDBCTypeConverter() self._formatter = formatter if formatter else ParameterFormatter()
def _create_client(self): session = botocore.session.get_session() if aws_creds: session.set_credentials(**aws_creds) else: session.set_config_variable('profile', self.profile) return session.create_client( self.service_name, region_name=self.region_name)
def gen_s3_obj(method): config = common.get_config() session = botocore.session.get_session() session.set_credentials(config["access_key"], config["secret_key"]) s3 = session.get_service("s3") operation = s3.get_operation(method) endpoint = s3.get_endpoint(config["region"]) return [endpoint, operation]
def gen_s3_obj(method): config = common.get_config() session = botocore.session.get_session() session.set_credentials(config['access_key'], config['secret_key']) s3 = session.get_service('s3') operation = s3.get_operation(method) endpoint = s3.get_endpoint(config['region']) return [endpoint, operation]
def setUp(self): super().setUp() BotocoreInstrumentor().instrument() session = botocore.session.get_session() session.set_credentials(access_key="access-key", secret_key="secret-key") self.client = session.create_client("dynamodb", region_name="us-west-2") self.default_table_name = "test_table"
def setUp(self): super().setUp() BotocoreInstrumentor().instrument() session = botocore.session.get_session() session.set_credentials( access_key="access-key", secret_key="secret-key" ) self.region = "us-west-2" self.client = session.create_client("lambda", region_name=self.region) self.iam_client = session.create_client("iam", region_name=self.region)
def _toggle_term_protect(name, enabled): ''' Toggle termination protection on a node ''' # region is required for all boto queries region = get_location(None) # init botocore vm_ = get_configured_provider() session = botocore.session.get_session() # pylint: disable=E0602 session.set_credentials( access_key=config.get_cloud_config_value( 'id', vm_, __opts__, search_global=False ), secret_key=config.get_cloud_config_value( 'key', vm_, __opts__, search_global=False ) ) service = session.get_service('ec2') endpoint = service.get_endpoint(region) # get the instance-id for the supplied node name conn = get_conn(location=region) node = get_node(conn, name) params = { 'instance_id': node.id, 'attribute': 'disableApiTermination', 'value': 'true' if enabled else 'false', } # get instance information operation = service.get_operation('modify-instance-attribute') http_response, response_data = operation.call(endpoint, **params) if http_response.status_code == 200: msg = 'Termination protection successfully {0} on {1}'.format( enabled and 'enabled' or 'disabled', name ) log.info(msg) return msg # No proper HTTP response!? msg = 'Bad response from AWS: {0}'.format(http_response.status_code) log.error(msg) return msg
def _toggle_term_protect(name, enabled): ''' Toggle termination protection on a node ''' # region is required for all boto queries region = get_location(None) # init botocore session = botocore.session.get_session() session.set_credentials( access_key=__opts__['AWS.id'], secret_key=__opts__['AWS.key'], ) service = session.get_service('ec2') endpoint = service.get_endpoint(region) # get the instance-id for the supplied node name conn = get_conn(location=region) node = get_node(conn, name) params = { 'instance_id': node.id, 'attribute': 'disableApiTermination', 'value': 'true' if enabled else 'false', } # get instance information operation = service.get_operation('modify-instance-attribute') http_response, response_data = operation.call(endpoint, **params) if http_response.status_code == 200: msg = ( 'Termination protection successfully {0} on {1}'.format( enabled and 'enabled' or 'disabled', name ) ) log.info(msg) return msg else: msg = ( 'Bad response from AWS: {0}'.format( http_response.status_code ) ) log.error(msg) return msg
def destroy_subnet(aws_creds, region, subnet_id=""): print('Destroying subnet %s' % str(subnet_id)) session = botocore.session.get_session() session.set_credentials(aws_creds.access_key, aws_creds.secret_key) ec2 = session.get_service('ec2') operation = ec2.get_operation('DeleteSubnet') endpoint = ec2.get_endpoint(region) http_response, response_data = operation.call(endpoint, SubnetId=subnet_id) p3.pprint(str(http_response.status_code) + " - " + http_response.reason) p3.pprint(response_data) if http_response.status_code != 200: raise(ApiException) return True
def mapper_init(self): self.named_passages_text = None try: if self.named_passages_filename and os.path.exists(os.path.join(os.getcwd(),self.named_passages_filename)): with open(os.path.join(os.getcwd(),self.named_passages_filename),'r') as named_passages_file: self.named_passages_text = named_passages_file.read() sys.stderr.write('named_passages_filename=%s size=%s\n'%(self.named_passages_filename,len(self.named_passages_text))) session = botocore.session.get_session() aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID') aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY') if aws_access_key_id == None: aws_access_key_id, aws_secret_access_key = self._get_s3_credentials() session.set_credentials(aws_access_key_id,aws_secret_access_key) self.s3_client = session.create_client('s3', region_name='us-east-1') except: sys.stderr.write(traceback.format_exc()+'\n')
def connect( cls, region: str, session: Optional[botocore.session.Session] = None, access_key: Optional[str] = None, secret_key: Optional[str] = None, host: Optional[str] = None, port: int = 80, is_secure: bool = True, dynamizer: Dynamizer = Dynamizer(), ) -> "DynamoDBConnection": """ Connect to an AWS region. Parameters ---------- region : str Name of an AWS region session : :class:`~botocore.session.Session`, optional The Session object to use for the connection access_key : str, optional If session is None, set this access key when creating the session secret_key : str, optional If session is None, set this secret key when creating the session host : str, optional Address of the host. Use this to connect to a local instance. port : int, optional Connect to the host on this port (default 80) is_secure : bool, optional Enforce https connection (default True) **kwargs : dict Keyword arguments to pass to the constructor """ if session is None: session = botocore.session.get_session() if access_key is not None: session.set_credentials(access_key, secret_key) url = None if host is not None: protocol = "https" if is_secure else "http" url = "%s://%s:%d" % (protocol, host, port) client = session.create_client("dynamodb", region, endpoint_url=url, use_ssl=is_secure) return cls(client, dynamizer)
def test_run(): logger = setup_custom_logger(__name__) agent = Agent() agent.instrument(None) # Setup In-Memory Span Exporter logger.info('Agent initialized.') logger.info('Adding in-memory span exporter.') memoryExporter = InMemorySpanExporter() simpleExportSpanProcessor = SimpleSpanProcessor(memoryExporter) agent.register_processor(simpleExportSpanProcessor) logger.info('Added in-memory span exporter') logger.info('Running test calls.') try: session = botocore.session.get_session() session.set_credentials(access_key="access-key", secret_key="secret-key") region = "us-west-2" client = session.create_client("lambda", region_name=region) iam_client = session.create_client("iam", region_name=region) arn = _create_role_and_get_arn(iam_client) result = _create_lambda_function('some_function', return_headers_lambda_str(), client, arn) memoryExporter.clear() response = client.invoke( Payload=json.dumps({}), FunctionName=result['FunctionArn'], InvocationType="RequestResponse", ) spans = memoryExporter.get_finished_spans() invoke_span = spans[-1] assert invoke_span.attributes['faas.invoked_name'] == 'some_function' assert invoke_span.attributes['http.status_code'] == 200 assert invoke_span.attributes['rpc.service'] == 'Lambda' memoryExporter.clear() return 0 except: logger.error( 'Failed to test boto instrumentation wrapper: exception=%s, stacktrace=%s', sys.exc_info()[0], traceback.format_exc()) raise sys.exc_info()[0]
def destroy_elb(aws_creds, region, lb_name=""): print('Destroying elb') session = botocore.session.get_session() session.set_credentials(aws_creds.access_key, aws_creds.secret_key) elb = session.get_service('elb') operation = elb.get_operation('DeleteLoadBalancer') endpoint = elb.get_endpoint(region) http_response, response_data = operation.call(endpoint, LoadBalancerName=lb_name) p3.pprint(str(http_response.status_code) + " - " + http_response.reason) p3.pprint(response_data) if http_response.status_code != 200: raise(ApiException) return True
def do_action(region, awsKey, awsSecret, ownerID, actionObject, actionName, objectUUID, parameters): session = botocore.session.get_session() session.set_credentials(outscaleKey, outscaleSecret) ec2 = session.get_service('ec2') if actionObject == 'instance': if actionName == 'start': try: # connEC2.start_instances(instance_ids=[objectUUID]) operation = ec2.get_operation('StartInstances'); http_response, response_data = operation.call(endpoint,instance_ids = [objectUUID]) return 200, "" except boto.exception.EC2ResponseError as e: description = "Unknown description" if len(e.errors): tmpVar = e.errors[0] if len(tmpVar): description = tmpVar[0] return e.status, description elif actionName == 'stop': try: operation = ec2.get_operation('StopInstances'); http_response, response_data = operation.call(endpoint,instance_ids = [objectUUID]) return 200, "" except boto.exception.EC2ResponseError as e: description = "Unknown description" if len(e.errors): tmpVar = e.errors[0] if len(tmpVar): description = tmpVar[0] return e.status, description elif actionObject == 'volume': if actionName == 'delete': try: operation = ec2.get_operation('DeleteVolume'); http_response, response_data = operation.call(endpoint,volume_ids = [objectUUID]) return 200,"" except boto.exception.EC2ResponseError as e: description = "Unknown description" if len(e.errors): tmpVar = e.errors[0] if len(tmpVar): description = tmpVar[0] return e.status, description pass;
def get_session(self, region_name, role_arn): """ Assumes the given role and returns a session object assuming said role. """ session = botocore.session.get_session() if region_name is not None: session.set_config_variable('region', region_name) if role_arn is not None: sts = session.create_client(AUTH_SERVICE, region_name=region_name) credentials_dict = sts.assume_role( RoleArn=role_arn, RoleSessionName='EKSGetTokenAuth')['Credentials'] session.set_credentials(credentials_dict['AccessKeyId'], credentials_dict['SecretAccessKey'], credentials_dict['SessionToken']) return session else: return session
def build_security_group(aws_creds, region, sg_name="", vpc_id=""): print('Building security group') session = botocore.session.get_session() session.set_credentials(aws_creds.access_key, aws_creds.secret_key) ec2 = session.get_service('ec2') operation = ec2.get_operation('CreateSecurityGroup') endpoint = ec2.get_endpoint(region) http_response, response_data = operation.call(endpoint, GroupName=sg_name, Description=sg_name, VpcId=vpc_id) p3.pprint(str(http_response.status_code) + " - " + http_response.reason) p3.pprint(response_data) if http_response.status_code != 200: raise(ApiException) sg_id = response_data['GroupId'] return sg_id
def build_subnet(aws_creds, region, vpc_id="", cidr="", az=""): print('Building subnet %s in AZ %s' % (str(cidr), str(az))) session = botocore.session.get_session() session.set_credentials(aws_creds.access_key, aws_creds.secret_key) ec2 = session.get_service('ec2') operation = ec2.get_operation('CreateSubnet') endpoint = ec2.get_endpoint(region) http_response, response_data = operation.call(endpoint, VpcId=vpc_id, CidrBlock=cidr, AvailabilityZone="%s%s" % ( region, az)) p3.pprint(str(http_response.status_code) + " - " + http_response.reason) p3.pprint(response_data) if http_response.status_code != 200: raise(ApiException) subnet_id = response_data['Subnet']['SubnetId'] return subnet_id
def get_session(self, region_name, role_arn): """ Assumes the given role and returns a session object assuming said role. """ session = botocore.session.get_session() if region_name is not None: session.set_config_variable('region', region_name) if role_arn is not None: sts = session.create_client(AUTH_SERVICE, region_name=region_name) credentials_dict = sts.assume_role( RoleArn=role_arn, RoleSessionName='EKSGetTokenAuth' )['Credentials'] session.set_credentials(credentials_dict['AccessKeyId'], credentials_dict['SecretAccessKey'], credentials_dict['SessionToken']) return session else: return session
def get_boto_session(region_name=None, profile=None, creds=None, cache=None): """ Get botocore.session with correct region_name configured """ if cache is not None: sessions = getattr(cache, 'sessions', None) if sessions is None: sessions = {} setattr(cache, 'sessions', sessions) session = sessions.get(region_name) else: sessions, session = {}, None if session is not None: return session session = botocore.session.Session(profile=profile) _region = session.get_config_variable("region") if creds is not None: session.set_credentials(creds.access_key, creds.secret_key, creds.token) if _region is None: if region_name is None or region_name == "auto": _region = auto_find_region(session) else: _region = region_name session.set_config_variable("region", _region) sessions[_region] = session return session
def _toggle_term_protect(name, enabled): """ Toggle termination protection on a node """ # region is required for all boto queries region = get_location(None) # init botocore vm_ = get_configured_provider() session = botocore.session.get_session() # pylint: disable=E0602 session.set_credentials( access_key=config.get_cloud_config_value("id", vm_, __opts__, search_global=False), secret_key=config.get_cloud_config_value("key", vm_, __opts__, search_global=False), ) service = session.get_service("ec2") endpoint = service.get_endpoint(region) # get the instance-id for the supplied node name conn = get_conn(location=region) node = get_node(conn, name) params = {"instance_id": node.id, "attribute": "disableApiTermination", "value": "true" if enabled else "false"} # get instance information operation = service.get_operation("modify-instance-attribute") http_response, response_data = operation.call(endpoint, **params) if http_response.status_code == 200: msg = "Termination protection successfully {0} on {1}".format(enabled and "enabled" or "disabled", name) log.info(msg) return msg # No proper HTTP response!? msg = "Bad response from AWS: {0}".format(http_response.status_code) log.error(msg) return msg
def connect(cls, region, session=None, access_key=None, secret_key=None, host=None, port=80, is_secure=True, **kwargs): """ Connect to an AWS region. Parameters ---------- region : str Name of an AWS region session : :class:`~botocore.session.Session`, optional The Session object to use for the connection access_key : str, optional If session is None, set this access key when creating the session secret_key : str, optional If session is None, set this secret key when creating the session host : str, optional Address of the host. Use this to connect to a local instance. port : int, optional Connect to the host on this port (default 80) is_secure : bool, optional Enforce https connection (default True) **kwargs : dict Keyword arguments to pass to the constructor """ if session is None: session = botocore.session.get_session() if access_key is not None: session.set_credentials(access_key, secret_key) service = session.get_service('dynamodb') url = None if host is not None: protocol = 'https' if is_secure else 'http' url = "%s://%s:%d" % (protocol, host, port) endpoint = service.get_endpoint(region, endpoint_url=url, is_secure=is_secure) return cls(service, endpoint, **kwargs)
def _initialize_from_config(self, beaver_config, filename): super(KmsEncrypter, self)._initialize_from_config(beaver_config, filename) if not aws_encryption_sdk: raise SystemError( 'The use_aws_kms_encryption option is only supported for python >= 2.7' ) kms_configs = _get_kms_config_values(beaver_config, filename) kms_secret_key = kms_configs.aws_kms_secret_key kms_access_key = kms_configs.aws_kms_access_key kms_key_ids = kms_configs.aws_kms_key_ids self.kms_encryption_context = kms_configs.aws_kms_encryption_context if not kms_key_ids: raise ValueError( 'You must provide at lease one CMS key for aws_kms_key_ids!') if self.kms_encryption_context and not isinstance( self.kms_encryption_context, dict): raise ValueError( 'kms_encryption_context must be a flat map of key value pairs, or not provided.' ) session = botocore.session.Session() if kms_access_key and kms_secret_key: session.set_credentials(kms_access_key, kms_secret_key) else: self.logger.info( 'Using default boto credentials locations for KMS access.') self.kms_client = aws_encryption_sdk.KMSMasterKeyProvider( botocore_session=session, key_ids=kms_key_ids)
def connect_to_host(cls, host='localhost', port=8000, is_secure=False, session=None, access_key=None, secret_key=None, **kwargs): """ Connect to a specific host. This method has been deprecated in favor of :meth:`~.connect` Parameters ---------- host : str, optional Address of the host (default 'localhost') port : int, optional Connect to the host on this port (default 8000) is_secure : bool, optional Enforce https connection (default False) session : :class:`~botocore.session.Session`, optional The Session object to use for the connection access_key : str, optional If session is None, set this access key when creating the session secret_key : str, optional If session is None, set this secret key when creating the session **kwargs : dict Keyword arguments to pass to the constructor """ warnings.warn("connect_to_host is deprecated and will be removed. " "Use connect instead.") if session is None: session = botocore.session.get_session() if access_key is not None: session.set_credentials(access_key, secret_key) url = "http://%s:%d" % (host, port) client = session.create_client('dynamodb', 'local', endpoint_url=url, use_ssl=is_secure) return cls(client, **kwargs)
def fetch_base_data(outscaleKey, outscaleSecret, outscaleName): all_regions = {"us-east-1":{'uuid':"us-east-1",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://api.outscale.us"}, \ "us-east-2":{'uuid':"us-east-2",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://fcu.us-east-2.outscale.com"}, \ "cn-southeast-1":{'uuid':"cn-southeast-1",'display_name':'China', 'zones':['a'], 'url':"https://fcu.cn-southeast-1.outscale.hk"}, \ "eu-west-1":{'uuid':"eu-west-1",'display_name':'France', 'zones':['a'], 'url':"https://api.outscale.com"}, \ "eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}}; all_regions = {"eu-west-1":{'uuid':"eu-west-1",'display_name':'France', 'zones':['a'], 'url':"https://api.outscale.com"},\ "us-east-1":{'uuid':"us-east-1",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://api.outscale.us"}, \ "us-east-2":{'uuid':"us-east-2",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://fcu.us-east-2.outscale.com"}, \ "eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}}; # all_regions = {"us-east-2":{'uuid':"us-east-2",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://fcu.us-east-2.outscale.com"},\ # "eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}, \ # "cn-southeast-1":{'uuid':"cn-southeast-1",'display_name':'China', 'zones':['a'], 'url':"https://fcu.cn-southeast-1.outscale.hk"}}; # all_regions = {"eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}}; # pp = pprint.PrettyPrinter(indent=4) pp.pprint(all_regions) session = botocore.session.get_session() ec2 = session.get_service('ec2') session.set_credentials(outscaleKey, outscaleSecret) # session.set_debug_logger() region_mapping_dict = dict(); all_objects = []; for a_region in all_regions: region_mapping_dict[a_region] = {'url':all_regions[a_region]['url'], 'objects':[], 'region':a_region, 'conn_status':{},'ec2': ec2, 'key':outscaleKey, 'secret':outscaleSecret} # print region_mapping_dict print "===" tsg = time.time() nep_log("Starting all regions fetch") jobs = [gevent.spawn(fetch_one_region, region_mapping_dict[a_region]) for a_region in all_regions] gevent.joinall(jobs, timeout=20) teg = time.time() nep_log("Done all regions fetch in "+str(teg-tsg)) pp = pprint.PrettyPrinter(indent=4) pp.pprint(region_mapping_dict) for a_region in all_regions: region_dict = dict() region_dict['__class_name'] = 'region' region_dict.update(all_regions[a_region]) region_dict['region'] = a_region all_objects.append(region_dict) for aZone in all_regions[a_region]['zones']: zone_dict = dict() zone_dict['__class_name'] = 'zone' zone_dict['region'] = a_region; zone_dict['uuid'] = region_dict['uuid']+aZone all_objects.append(zone_dict) tmpUUID = 'nep.type.aws.outscale-'+str(uuid.uuid1()) connection_shortDict = dict() connection_shortDict['__class_name'] = 'connection' connection_shortDict['name'] = outscaleName connection_shortDict['nep_connection_type'] = 'nep.type.aws.outscale' connection_shortDict['nep_connection_family'] = 'nep.family.aws' connection_shortDict['uuid'] = tmpUUID connection_shortDict['key'] = outscaleKey connection_shortDict['secret'] = outscaleSecret all_objects.append(connection_shortDict) hasAtLeastOneError=False; hasAllError=True; hasOneRegionAllErrors = True; hasRegionAllErrors = True isErrorRelevant = False; regionErrorsStatus = {} regionErrorGlobalCount = 0 for key in region_mapping_dict: regionErrorGlobalCount = regionErrorGlobalCount+ 1; regionErrorsStatus[key] = {'allerror':True, 'oneerror':False, 'errors':[]} allErrors = [] for key in region_mapping_dict: for anObj in region_mapping_dict[key]['objects']: all_objects.append(anObj); for anObj in region_mapping_dict[key]['conn_status']: endStatus = region_mapping_dict[key]['conn_status'][anObj] # pp.pprint(region_mapping_dict[key]['conn_status']) if endStatus['status'] == 200: regionErrorsStatus[key]['allerror'] = False # hasAllError = False; else: regionErrorsStatus[key]['oneerror'] = True; # hasAtLeastOneError = True; if str(endStatus['status']) not in regionErrorsStatus[key]['errors']: regionErrorsStatus[key]['errors'].append(str(endStatus['status'])) if regionErrorsStatus[key]['allerror'] == True: regionErrorsStatus[key]['oneerror'] = False #For outscale if a region is all false this is due to the authorization problem # if regionHasAllError == True: # hasAtLeastOneError = False; # print "ALLERROR" connection_dict = dict() connection_dict['__class_name'] = 'connection' connection_dict['name'] = outscaleName connection_dict['nep_connection_type'] = 'nep.type.aws.outscale' connection_dict['nep_connection_family'] = 'nep.family.aws' connection_dict['uuid'] = tmpUUID connection_dict['objects'] = all_objects connection_dict['key'] = outscaleKey connection_dict['secret'] = outscaleSecret hasOneRelevantPartielError = False regionErrorFinalCount = 0 for key in region_mapping_dict: if regionErrorsStatus[key]['oneerror'] == True: isErrorRelevant = True; if regionErrorsStatus[key]['allerror'] == True: regionErrorFinalCount = regionErrorFinalCount+1 if isErrorRelevant == False: if regionErrorFinalCount == regionErrorGlobalCount: isErrorRelevant = True hasAllError = True; if isErrorRelevant: for key in region_mapping_dict: for anError in regionErrorsStatus[key]['errors']: if anError not in allErrors: allErrors.append(anError) if isErrorRelevant: errorDict = {} errorDict['__class_name'] = 'error' errorDict['uuid'] = 'error_'+str(uuid.uuid1()) errorDict['codes'] = ','.join(allErrors) if hasAllError: errorDict['partial'] = False errorDict['message'] = 'An error has occured please check that you have proper IAM credentials to access the infrastructure data and that credentials are properly entered' else: errorDict['partial'] = True errorDict['message'] = 'Some errors have occured please check that you have proper IAM credentials to access the infrastructure data.' errorDict['partial'] = True connection_dict['error'] = errorDict; # pp = pprint.PrettyPrinter(indent=4) # pp.pprint(connection_dict) return connection_dict
import json import botocore.session from botocore.exceptions import ClientError from scrapy.conf import settings session = botocore.session.get_session() session.set_credentials(access_key=settings['AWS_ACCESS_KEY'], secret_key=settings['AWS_SECRET_KEY']) client = session.create_client('s3', region_name=settings['S3_REGION']) def load_members(): try: response = client.get_object(Bucket=settings['S3_BUCKET'], Key=settings['S3_KEY']) members = json.loads(response['Body'].read()) except ClientError: members = {} return members def save_members(members): client.put_object(ACL='private', Bucket=settings['S3_BUCKET'], Key=settings['S3_KEY'], Body=json.dumps(members))
def main(): # # parse args # parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', help='verbose output', action='store_true') parser.add_argument('command', help='command passed to aws cli', nargs='+') args = parser.parse_args() command = ' '.join(args.command) credentials_file = str( os.getenv('AWS_ENCRYPTED_CREDENTIALS_FILE', '~/.aws/credentials.gpg')) config_file = str(os.getenv('AWS_CONFIG_FILE', '~/.aws/config')) profile = str( os.getenv('AWS_PROFILE', os.getenv('AWS_DEFAULT_PROFILE', 'default'))) if args.verbose: print 'AWS_ENCRYPTED_CREDENTIALS_FILE={}'.format(credentials_file) print 'AWS_CONFIG_FILE={}'.format(config_file) print 'AWS_DEFAULT_PROFILE={}'.format(profile) print 'command:', command # # parse config # config = ConfigParser.ConfigParser() config.read(os.path.expanduser(config_file)) config_section = 'profile ' + profile source_profile = profile role_arn = None if config.has_option(config_section, 'source_profile'): source_profile = config.get(config_section, 'source_profile') if config.has_option(config_section, 'role_arn'): role_arn = config.get(config_section, 'role_arn') if args.verbose: print 'source_profile=' + str(source_profile) print 'role_arn=' + str(role_arn) # # read encrypted file # credentials_file = os.path.expanduser(credentials_file) if not os.path.exists(credentials_file): sys.exit( 'Unable to find credentials file, {}'.format(credentials_file)) try: with open(credentials_file, mode='rb') as file: encrypted = file.read() except IOError as e: sys.exit('I/O error({0}): {1}; credentials_file: {2}'.format( e.errno, e.strerror, credentials_file)) except: sys.exit('Unexpected error:'.format(sys.exc_info()[0])) # # gpgme # gme = gpgme.Context() encrypted_bytes = io.BytesIO(encrypted) decrypted_bytes = io.BytesIO() try: gme.decrypt(encrypted_bytes, decrypted_bytes) except gpgme.GpgmeError as e: sys.exit('Unable to decrypt file; {}'.format(e.strerror)) decrypted_string = decrypted_bytes.getvalue().decode('utf8') # # parse credentials # # make sure to read all of decrypted_bytes decrypted_bytes.seek(0) credentials = ConfigParser.ConfigParser() credentials.readfp(decrypted_bytes) if not credentials.has_section(source_profile): sys.exit( 'Credentials file does not have source_profile "{}", AWS_PROFILE={}' .format(source_profile, profile)) if not credentials.has_option(source_profile, 'aws_access_key_id'): sys.exit( 'Credentials file source_profile does not have aws_access_key_id, {}' .format(source_profile)) if not credentials.has_option(source_profile, 'aws_secret_access_key'): sys.exit( 'Credentials file source_profile does not have aws_secret_access_key, {}' .format(source_profile)) aws_access_key_id = credentials.get(source_profile, 'aws_access_key_id') aws_secret_access_key = credentials.get(source_profile, 'aws_secret_access_key') # # setup a botocore session # session = botocore.session.Session(profile=profile) session.set_credentials(aws_access_key_id, aws_secret_access_key) try: scoped_config = session.get_scoped_config() region = scoped_config.get('region', None) if region: os.putenv('AWS_DEFAULT_REGION', region) os.putenv('AWS_REGION', region) except botocore.exceptions.ProfileNotFound as e: sys.exit('Profile not found in config; profile={}'.format(profile)) if args.verbose: print 'AWS_DEFAULT_REGION={}'.format(region) # # switch iam roles using sts # if role_arn == None: os.putenv('AWS_ACCESS_KEY_ID', aws_access_key_id) os.putenv('AWS_SECRET_ACCESS_KEY', aws_secret_access_key) else: sts_client = session.create_client('sts') assumed_role_object = sts_client.assume_role( RoleArn=role_arn, RoleSessionName='AssumeRoleSession1') role_credentials = assumed_role_object['Credentials'] os.putenv('AWS_ACCESS_KEY_ID', role_credentials['AccessKeyId']) os.putenv('AWS_SECRET_ACCESS_KEY', role_credentials['SecretAccessKey']) os.putenv('AWS_SESSION_TOKEN', role_credentials['SessionToken']) # # run the command # if args.verbose: print 'Command output:' my_env = os.environ.copy() command_status = os.system(command) exit(os.WEXITSTATUS(command_status))