def connect_to_region(cls, region, session=None, access_key=None,
secret_key=None, **kwargs):
"""
Connect to an AWS region.
This method has been deprecated in favor of :meth:`~.connect`
Parameters
----------
region : str
Name of an AWS region
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
**kwargs : dict
Keyword arguments to pass to the constructor
"""
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
service = session.get_service('dynamodb')
return cls(service, service.get_endpoint(region), **kwargs)
def upload_to_sandbox(path):
print('File \'%s\' uploading started...' % path.replace('api/test/../../', ''))
start_time = str(int(time.time()))
session = botocore.session.get_session()
session.set_credentials(access_key="AKIAIA5IPHVWBGQSSBBA",
secret_key="uhsuCK/U8hBWYkMHPoB9dGFJHxjTFK4y63cx3D/y")
client = session.create_client('s3', region_name='us-west-2')
f = open(path, 'rb')
key = '/'.join(unicode(uuid4()).split('-') + [os.path.basename(path)])
response = client.put_object(
Body=f.read(),
Bucket='bkstg-sandbox',
Key=key,
ContentType=mimetypes.guess_type(path)[0]
)
end_time = str(int(time.time()))
print('File was uploaded in %s seconds.\n' % (int(end_time) - int(start_time)))
return key
# if __name__ == '__main__':
# Upload.upload_to_sandbox(path=TestData.image_path)
def mk_boto_session(
profile: Optional[str] = None,
creds: Optional[ReadOnlyCredentials] = None,
region_name: Optional[str] = None,
) -> Session:
"""Get botocore session with correct `region` configured
:param profile: profile name to lookup
:param creds: Override credentials with supplied data
:param region_name: default region_name to use if not configured for a given profile
"""
session = botocore.session.Session(profile=profile)
if creds is not None:
session.set_credentials(creds.access_key, creds.secret_key, creds.token)
_region = session.get_config_variable("region")
if _region is None:
if region_name is None or region_name == "auto":
_region = auto_find_region(session, default="us-west-2")
else:
_region = region_name
session.set_config_variable("region", _region)
return session
def build_vpc(aws_creds, region, cidr=""):
print('Building VPC for %s network' % str(cidr))
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
ec2 = session.get_service('ec2')
operation = ec2.get_operation('CreateVpc')
endpoint = ec2.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
CidrBlock=cidr)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
vpc_id = response_data['Vpc']['VpcId']
operation = ec2.get_operation('CreateTags')
http_response, response_data = operation.call(endpoint,
Resources=[vpc_id],
Tags=[{"Key": "Name",
"Value": 'Meetup'
}],
)
return vpc_id
def main():
session = botocore.session.Session()
session.set_credentials(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY)
ec2 = session.get_service('ec2')
endpoint = ec2.get_endpoint(AWS_DEFAULT_REGION)
op_din = ec2.get_operation('DescribeInstances')
op_stop = ec2.get_operation('StopInstances')
http_response, reservation_data = op_din.call(endpoint, filters=[{"Name":"instance-state-name","Values":["running"]}] )
if not http_response.ok:
print(reservation_data['Errors'])
sys.exit(2)
else:
reservations = reservation_data['Reservations']
#Pull the instances out of reservations
instances = []
for reservation in reservations:
instances += reservation['Instances']
print("Found %i instances in running state" % len(instances))
instance_ids = [instance['InstanceId'] for instance in instances]
http_response, data = op_stop.call(endpoint, instance_ids = instance_ids)
if not http_response.ok:
print(data['Errors'])
sys.exit(2)
for instance in data['StoppingInstances']:
print("Instance %s now stopping" % instance['InstanceId'])
def connect_to_host(cls, host='localhost', port=8000, is_secure=False,
session=None, access_key=None, secret_key=None,
**kwargs):
"""
Connect to a specific host.
This method has been deprecated in favor of :meth:`~.connect`
Parameters
----------
host : str, optional
Address of the host (default 'localhost')
port : int, optional
Connect to the host on this port (default 8000)
is_secure : bool, optional
Enforce https connection (default False)
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
**kwargs : dict
Keyword arguments to pass to the constructor
"""
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
url = "http://%s:%d" % (host, port)
service = session.get_service('dynamodb')
endpoint = service.get_endpoint(
'local', endpoint_url=url, is_secure=is_secure)
return cls(service, endpoint, **kwargs)
def connect_to_region(cls, region, session=None, access_key=None,
secret_key=None, **kwargs):
"""
Connect to an AWS region.
This method has been deprecated in favor of :meth:`~.connect`
Parameters
----------
region : str
Name of an AWS region
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
**kwargs : dict
Keyword arguments to pass to the constructor
"""
warnings.warn("connect_to_region is deprecated and will be removed. "
"Use connect instead.")
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
client = session.create_client('dynamodb', region)
return cls(client, **kwargs)
def build_elb(aws_creds, region, lb_name="", subnets=[], elb_sg_id=""):
print('Building elb')
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
elb = session.get_service('elb')
operation = elb.get_operation('CreateLoadBalancer')
endpoint = elb.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
LoadBalancerName=lb_name,
Listeners=[
{"Protocol": 'http',
"LoadBalancerPort": 80,
"InstanceProtocol": 'http',
"InstancePort": 80}],
Subnets=subnets,
SecurityGroups=[elb_sg_id],
Scheme='internal')
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
elb_name = response_data['DNSName']
return elb_name
def boto3_session(profile_name=None,
config_file=None,
credentials_file=None,
existing_creds=None):
"""
returns a boto3 session using cached credentials
Allows for sharing credentials from cli commands with boto sessions
"""
# Construct botocore session with cache
session = botocore.session.get_session()
if profile_name is not None:
session.set_config_variable("profile", profile_name)
if config_file is not None:
session.set_config_variable("config_file", config_file)
if credentials_file is not None:
session.set_config_variable("credentials_file", credentials_file)
if existing_creds is not None:
session.set_credentials(
access_key=existing_creds["aws_access_key_id"],
secret_key=existing_creds["aws_secret_access_key"],
token=existing_creds["aws_session_token"],
)
session.get_component("credential_provider").get_provider(
"assume-role").cache = credentials.JSONFileCache(cli_cache)
return boto3.Session(botocore_session=session)
def __init__(self,
s3_staging_dir=None,
access_key=None,
secret_key=None,
region_name=None,
schema_name='default',
profile_name=None,
credential_file=None,
jvm_path=None,
jvm_options=None,
converter=None,
formatter=None,
driver_path=None,
**driver_kwargs):
if s3_staging_dir:
self.s3_staging_dir = s3_staging_dir
else:
self.s3_staging_dir = os.getenv(self._ENV_S3_STAGING_DIR, None)
assert self.s3_staging_dir, 'Required argument `s3_staging_dir` not found.'
assert schema_name, 'Required argument `schema_name` not found.'
self.schema_name = schema_name
if credential_file:
self.access_key = None
self.secret_key = None
self.token = None
self.credential_file = credential_file
assert self.credential_file, 'Required argument `credential_file` not found.'
self.region_name = region_name
assert self.region_name, 'Required argument `region_name` not found.'
else:
import botocore.session
session = botocore.session.get_session()
if access_key and secret_key:
session.set_credentials(access_key, secret_key)
if profile_name:
session.set_config_variable('profile', profile_name)
if region_name:
session.set_config_variable('region', region_name)
credentials = session.get_credentials()
self.access_key = credentials.access_key
assert self.access_key, 'Required argument `access_key` not found.'
self.secret_key = credentials.secret_key
assert self.secret_key, 'Required argument `secret_key` not found.'
self.token = credentials.token
self.credential_file = None
self.region_name = session.get_config_variable('region')
assert self.region_name, 'Required argument `region_name` not found.'
self._start_jvm(jvm_path, jvm_options, driver_path)
props = self._build_driver_args(**driver_kwargs)
jpype.JClass(ATHENA_DRIVER_CLASS_NAME)
self._jdbc_conn = jpype.java.sql.DriverManager.getConnection(
ATHENA_CONNECTION_STRING.format(region=self.region_name,
schema=schema_name), props)
self._converter = converter if converter else JDBCTypeConverter()
self._formatter = formatter if formatter else ParameterFormatter()
def _create_client(self):
session = botocore.session.get_session()
if aws_creds:
session.set_credentials(**aws_creds)
else:
session.set_config_variable('profile', self.profile)
return session.create_client(
self.service_name, region_name=self.region_name)
def gen_s3_obj(method):
config = common.get_config()
session = botocore.session.get_session()
session.set_credentials(config["access_key"], config["secret_key"])
s3 = session.get_service("s3")
operation = s3.get_operation(method)
endpoint = s3.get_endpoint(config["region"])
return [endpoint, operation]
def gen_s3_obj(method):
config = common.get_config()
session = botocore.session.get_session()
session.set_credentials(config['access_key'], config['secret_key'])
s3 = session.get_service('s3')
operation = s3.get_operation(method)
endpoint = s3.get_endpoint(config['region'])
return [endpoint, operation]
def setUp(self):
super().setUp()
BotocoreInstrumentor().instrument()
session = botocore.session.get_session()
session.set_credentials(access_key="access-key",
secret_key="secret-key")
self.client = session.create_client("dynamodb",
region_name="us-west-2")
self.default_table_name = "test_table"
def setUp(self):
super().setUp()
BotocoreInstrumentor().instrument()
session = botocore.session.get_session()
session.set_credentials(
access_key="access-key", secret_key="secret-key"
)
self.region = "us-west-2"
self.client = session.create_client("lambda", region_name=self.region)
self.iam_client = session.create_client("iam", region_name=self.region)
def _toggle_term_protect(name, enabled):
'''
Toggle termination protection on a node
'''
# region is required for all boto queries
region = get_location(None)
# init botocore
vm_ = get_configured_provider()
session = botocore.session.get_session() # pylint: disable=E0602
session.set_credentials(
access_key=config.get_cloud_config_value(
'id', vm_, __opts__, search_global=False
),
secret_key=config.get_cloud_config_value(
'key', vm_, __opts__, search_global=False
)
)
service = session.get_service('ec2')
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {
'instance_id': node.id,
'attribute': 'disableApiTermination',
'value': 'true' if enabled else 'false',
}
# get instance information
operation = service.get_operation('modify-instance-attribute')
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = 'Termination protection successfully {0} on {1}'.format(
enabled and 'enabled' or 'disabled',
name
)
log.info(msg)
return msg
# No proper HTTP response!?
msg = 'Bad response from AWS: {0}'.format(http_response.status_code)
log.error(msg)
return msg
def _toggle_term_protect(name, enabled):
'''
Toggle termination protection on a node
'''
# region is required for all boto queries
region = get_location(None)
# init botocore
session = botocore.session.get_session()
session.set_credentials(
access_key=__opts__['AWS.id'],
secret_key=__opts__['AWS.key'],
)
service = session.get_service('ec2')
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {
'instance_id': node.id,
'attribute': 'disableApiTermination',
'value': 'true' if enabled else 'false',
}
# get instance information
operation = service.get_operation('modify-instance-attribute')
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = (
'Termination protection successfully {0} on {1}'.format(
enabled and 'enabled' or 'disabled',
name
)
)
log.info(msg)
return msg
else:
msg = (
'Bad response from AWS: {0}'.format(
http_response.status_code
)
)
log.error(msg)
return msg
def _toggle_term_protect(name, enabled):
'''
Toggle termination protection on a node
'''
# region is required for all boto queries
region = get_location(None)
# init botocore
vm_ = get_configured_provider()
session = botocore.session.get_session() # pylint: disable=E0602
session.set_credentials(
access_key=config.get_cloud_config_value(
'id', vm_, __opts__, search_global=False
),
secret_key=config.get_cloud_config_value(
'key', vm_, __opts__, search_global=False
)
)
service = session.get_service('ec2')
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {
'instance_id': node.id,
'attribute': 'disableApiTermination',
'value': 'true' if enabled else 'false',
}
# get instance information
operation = service.get_operation('modify-instance-attribute')
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = 'Termination protection successfully {0} on {1}'.format(
enabled and 'enabled' or 'disabled',
name
)
log.info(msg)
return msg
# No proper HTTP response!?
msg = 'Bad response from AWS: {0}'.format(http_response.status_code)
log.error(msg)
return msg
def destroy_subnet(aws_creds, region, subnet_id=""):
print('Destroying subnet %s' % str(subnet_id))
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
ec2 = session.get_service('ec2')
operation = ec2.get_operation('DeleteSubnet')
endpoint = ec2.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
SubnetId=subnet_id)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
return True
def mapper_init(self):
self.named_passages_text = None
try:
if self.named_passages_filename and os.path.exists(os.path.join(os.getcwd(),self.named_passages_filename)):
with open(os.path.join(os.getcwd(),self.named_passages_filename),'r') as named_passages_file:
self.named_passages_text = named_passages_file.read()
sys.stderr.write('named_passages_filename=%s size=%s\n'%(self.named_passages_filename,len(self.named_passages_text)))
session = botocore.session.get_session()
aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID')
aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if aws_access_key_id == None:
aws_access_key_id, aws_secret_access_key = self._get_s3_credentials()
session.set_credentials(aws_access_key_id,aws_secret_access_key)
self.s3_client = session.create_client('s3', region_name='us-east-1')
except:
sys.stderr.write(traceback.format_exc()+'\n')
def connect(
cls,
region: str,
session: Optional[botocore.session.Session] = None,
access_key: Optional[str] = None,
secret_key: Optional[str] = None,
host: Optional[str] = None,
port: int = 80,
is_secure: bool = True,
dynamizer: Dynamizer = Dynamizer(),
) -> "DynamoDBConnection":
"""
Connect to an AWS region.
Parameters
----------
region : str
Name of an AWS region
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
host : str, optional
Address of the host. Use this to connect to a local instance.
port : int, optional
Connect to the host on this port (default 80)
is_secure : bool, optional
Enforce https connection (default True)
**kwargs : dict
Keyword arguments to pass to the constructor
"""
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
url = None
if host is not None:
protocol = "https" if is_secure else "http"
url = "%s://%s:%d" % (protocol, host, port)
client = session.create_client("dynamodb",
region,
endpoint_url=url,
use_ssl=is_secure)
return cls(client, dynamizer)
def test_run():
logger = setup_custom_logger(__name__)
agent = Agent()
agent.instrument(None)
# Setup In-Memory Span Exporter
logger.info('Agent initialized.')
logger.info('Adding in-memory span exporter.')
memoryExporter = InMemorySpanExporter()
simpleExportSpanProcessor = SimpleSpanProcessor(memoryExporter)
agent.register_processor(simpleExportSpanProcessor)
logger.info('Added in-memory span exporter')
logger.info('Running test calls.')
try:
session = botocore.session.get_session()
session.set_credentials(access_key="access-key",
secret_key="secret-key")
region = "us-west-2"
client = session.create_client("lambda", region_name=region)
iam_client = session.create_client("iam", region_name=region)
arn = _create_role_and_get_arn(iam_client)
result = _create_lambda_function('some_function',
return_headers_lambda_str(), client,
arn)
memoryExporter.clear()
response = client.invoke(
Payload=json.dumps({}),
FunctionName=result['FunctionArn'],
InvocationType="RequestResponse",
)
spans = memoryExporter.get_finished_spans()
invoke_span = spans[-1]
assert invoke_span.attributes['faas.invoked_name'] == 'some_function'
assert invoke_span.attributes['http.status_code'] == 200
assert invoke_span.attributes['rpc.service'] == 'Lambda'
memoryExporter.clear()
return 0
except:
logger.error(
'Failed to test boto instrumentation wrapper: exception=%s, stacktrace=%s',
sys.exc_info()[0], traceback.format_exc())
raise sys.exc_info()[0]
def destroy_elb(aws_creds, region, lb_name=""):
print('Destroying elb')
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
elb = session.get_service('elb')
operation = elb.get_operation('DeleteLoadBalancer')
endpoint = elb.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
LoadBalancerName=lb_name)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
return True
def do_action(region, awsKey, awsSecret, ownerID, actionObject, actionName, objectUUID, parameters):
session = botocore.session.get_session()
session.set_credentials(outscaleKey, outscaleSecret)
ec2 = session.get_service('ec2')
if actionObject == 'instance':
if actionName == 'start':
try:
# connEC2.start_instances(instance_ids=[objectUUID])
operation = ec2.get_operation('StartInstances');
http_response, response_data = operation.call(endpoint,instance_ids = [objectUUID])
return 200, ""
except boto.exception.EC2ResponseError as e:
description = "Unknown description"
if len(e.errors):
tmpVar = e.errors[0]
if len(tmpVar):
description = tmpVar[0]
return e.status, description
elif actionName == 'stop':
try:
operation = ec2.get_operation('StopInstances');
http_response, response_data = operation.call(endpoint,instance_ids = [objectUUID])
return 200, ""
except boto.exception.EC2ResponseError as e:
description = "Unknown description"
if len(e.errors):
tmpVar = e.errors[0]
if len(tmpVar):
description = tmpVar[0]
return e.status, description
elif actionObject == 'volume':
if actionName == 'delete':
try:
operation = ec2.get_operation('DeleteVolume');
http_response, response_data = operation.call(endpoint,volume_ids = [objectUUID])
return 200,""
except boto.exception.EC2ResponseError as e:
description = "Unknown description"
if len(e.errors):
tmpVar = e.errors[0]
if len(tmpVar):
description = tmpVar[0]
return e.status, description
pass;
def get_session(self, region_name, role_arn):
"""
Assumes the given role and returns a session object assuming said role.
"""
session = botocore.session.get_session()
if region_name is not None:
session.set_config_variable('region', region_name)
if role_arn is not None:
sts = session.create_client(AUTH_SERVICE, region_name=region_name)
credentials_dict = sts.assume_role(
RoleArn=role_arn,
RoleSessionName='EKSGetTokenAuth')['Credentials']
session.set_credentials(credentials_dict['AccessKeyId'],
credentials_dict['SecretAccessKey'],
credentials_dict['SessionToken'])
return session
else:
return session
def build_security_group(aws_creds, region, sg_name="", vpc_id=""):
print('Building security group')
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
ec2 = session.get_service('ec2')
operation = ec2.get_operation('CreateSecurityGroup')
endpoint = ec2.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
GroupName=sg_name,
Description=sg_name,
VpcId=vpc_id)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
sg_id = response_data['GroupId']
return sg_id
def build_subnet(aws_creds, region, vpc_id="", cidr="", az=""):
print('Building subnet %s in AZ %s' % (str(cidr), str(az)))
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
ec2 = session.get_service('ec2')
operation = ec2.get_operation('CreateSubnet')
endpoint = ec2.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
VpcId=vpc_id,
CidrBlock=cidr,
AvailabilityZone="%s%s" % (
region, az))
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
subnet_id = response_data['Subnet']['SubnetId']
return subnet_id
def get_session(self, region_name, role_arn):
"""
Assumes the given role and returns a session object assuming said role.
"""
session = botocore.session.get_session()
if region_name is not None:
session.set_config_variable('region', region_name)
if role_arn is not None:
sts = session.create_client(AUTH_SERVICE, region_name=region_name)
credentials_dict = sts.assume_role(
RoleArn=role_arn,
RoleSessionName='EKSGetTokenAuth'
)['Credentials']
session.set_credentials(credentials_dict['AccessKeyId'],
credentials_dict['SecretAccessKey'],
credentials_dict['SessionToken'])
return session
else:
return session
def get_boto_session(region_name=None,
profile=None,
creds=None,
cache=None):
""" Get botocore.session with correct region_name configured
"""
if cache is not None:
sessions = getattr(cache, 'sessions', None)
if sessions is None:
sessions = {}
setattr(cache, 'sessions', sessions)
session = sessions.get(region_name)
else:
sessions, session = {}, None
if session is not None:
return session
session = botocore.session.Session(profile=profile)
_region = session.get_config_variable("region")
if creds is not None:
session.set_credentials(creds.access_key,
creds.secret_key,
creds.token)
if _region is None:
if region_name is None or region_name == "auto":
_region = auto_find_region(session)
else:
_region = region_name
session.set_config_variable("region", _region)
sessions[_region] = session
return session
def _toggle_term_protect(name, enabled):
"""
Toggle termination protection on a node
"""
# region is required for all boto queries
region = get_location(None)
# init botocore
vm_ = get_configured_provider()
session = botocore.session.get_session() # pylint: disable=E0602
session.set_credentials(
access_key=config.get_cloud_config_value("id", vm_, __opts__, search_global=False),
secret_key=config.get_cloud_config_value("key", vm_, __opts__, search_global=False),
)
service = session.get_service("ec2")
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {"instance_id": node.id, "attribute": "disableApiTermination", "value": "true" if enabled else "false"}
# get instance information
operation = service.get_operation("modify-instance-attribute")
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = "Termination protection successfully {0} on {1}".format(enabled and "enabled" or "disabled", name)
log.info(msg)
return msg
# No proper HTTP response!?
msg = "Bad response from AWS: {0}".format(http_response.status_code)
log.error(msg)
return msg
def connect(cls, region, session=None, access_key=None, secret_key=None,
host=None, port=80, is_secure=True, **kwargs):
"""
Connect to an AWS region.
Parameters
----------
region : str
Name of an AWS region
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
host : str, optional
Address of the host. Use this to connect to a local instance.
port : int, optional
Connect to the host on this port (default 80)
is_secure : bool, optional
Enforce https connection (default True)
**kwargs : dict
Keyword arguments to pass to the constructor
"""
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
service = session.get_service('dynamodb')
url = None
if host is not None:
protocol = 'https' if is_secure else 'http'
url = "%s://%s:%d" % (protocol, host, port)
endpoint = service.get_endpoint(region, endpoint_url=url,
is_secure=is_secure)
return cls(service, endpoint, **kwargs)
def _initialize_from_config(self, beaver_config, filename):
super(KmsEncrypter,
self)._initialize_from_config(beaver_config, filename)
if not aws_encryption_sdk:
raise SystemError(
'The use_aws_kms_encryption option is only supported for python >= 2.7'
)
kms_configs = _get_kms_config_values(beaver_config, filename)
kms_secret_key = kms_configs.aws_kms_secret_key
kms_access_key = kms_configs.aws_kms_access_key
kms_key_ids = kms_configs.aws_kms_key_ids
self.kms_encryption_context = kms_configs.aws_kms_encryption_context
if not kms_key_ids:
raise ValueError(
'You must provide at lease one CMS key for aws_kms_key_ids!')
if self.kms_encryption_context and not isinstance(
self.kms_encryption_context, dict):
raise ValueError(
'kms_encryption_context must be a flat map of key value pairs, or not provided.'
)
session = botocore.session.Session()
if kms_access_key and kms_secret_key:
session.set_credentials(kms_access_key, kms_secret_key)
else:
self.logger.info(
'Using default boto credentials locations for KMS access.')
self.kms_client = aws_encryption_sdk.KMSMasterKeyProvider(
botocore_session=session, key_ids=kms_key_ids)
def connect_to_host(cls, host='localhost', port=8000, is_secure=False,
session=None, access_key=None, secret_key=None,
**kwargs):
"""
Connect to a specific host.
This method has been deprecated in favor of :meth:`~.connect`
Parameters
----------
host : str, optional
Address of the host (default 'localhost')
port : int, optional
Connect to the host on this port (default 8000)
is_secure : bool, optional
Enforce https connection (default False)
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
**kwargs : dict
Keyword arguments to pass to the constructor
"""
warnings.warn("connect_to_host is deprecated and will be removed. "
"Use connect instead.")
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
url = "http://%s:%d" % (host, port)
client = session.create_client('dynamodb', 'local', endpoint_url=url,
use_ssl=is_secure)
return cls(client, **kwargs)
def fetch_base_data(outscaleKey, outscaleSecret, outscaleName):
all_regions = {"us-east-1":{'uuid':"us-east-1",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://api.outscale.us"}, \
"us-east-2":{'uuid':"us-east-2",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://fcu.us-east-2.outscale.com"}, \
"cn-southeast-1":{'uuid':"cn-southeast-1",'display_name':'China', 'zones':['a'], 'url':"https://fcu.cn-southeast-1.outscale.hk"}, \
"eu-west-1":{'uuid':"eu-west-1",'display_name':'France', 'zones':['a'], 'url':"https://api.outscale.com"}, \
"eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}};
all_regions = {"eu-west-1":{'uuid':"eu-west-1",'display_name':'France', 'zones':['a'], 'url':"https://api.outscale.com"},\
"us-east-1":{'uuid':"us-east-1",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://api.outscale.us"}, \
"us-east-2":{'uuid':"us-east-2",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://fcu.us-east-2.outscale.com"}, \
"eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}};
# all_regions = {"us-east-2":{'uuid':"us-east-2",'display_name':'N. Virginia', 'zones':['a'], 'url':"https://fcu.us-east-2.outscale.com"},\
# "eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}, \
# "cn-southeast-1":{'uuid':"cn-southeast-1",'display_name':'China', 'zones':['a'], 'url':"https://fcu.cn-southeast-1.outscale.hk"}};
# all_regions = {"eu-west-2":{'uuid':"eu-west-2",'display_name':'France', 'zones':['a'], 'url':"https://fcu.eu-west-2.outscale.com"}};
#
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(all_regions)
session = botocore.session.get_session()
ec2 = session.get_service('ec2')
session.set_credentials(outscaleKey, outscaleSecret)
# session.set_debug_logger()
region_mapping_dict = dict();
all_objects = [];
for a_region in all_regions:
region_mapping_dict[a_region] = {'url':all_regions[a_region]['url'], 'objects':[], 'region':a_region, 'conn_status':{},'ec2': ec2, 'key':outscaleKey, 'secret':outscaleSecret}
# print region_mapping_dict
print "==="
tsg = time.time()
nep_log("Starting all regions fetch")
jobs = [gevent.spawn(fetch_one_region, region_mapping_dict[a_region]) for a_region in all_regions]
gevent.joinall(jobs, timeout=20)
teg = time.time()
nep_log("Done all regions fetch in "+str(teg-tsg))
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(region_mapping_dict)
for a_region in all_regions:
region_dict = dict()
region_dict['__class_name'] = 'region'
region_dict.update(all_regions[a_region])
region_dict['region'] = a_region
all_objects.append(region_dict)
for aZone in all_regions[a_region]['zones']:
zone_dict = dict()
zone_dict['__class_name'] = 'zone'
zone_dict['region'] = a_region;
zone_dict['uuid'] = region_dict['uuid']+aZone
all_objects.append(zone_dict)
tmpUUID = 'nep.type.aws.outscale-'+str(uuid.uuid1())
connection_shortDict = dict()
connection_shortDict['__class_name'] = 'connection'
connection_shortDict['name'] = outscaleName
connection_shortDict['nep_connection_type'] = 'nep.type.aws.outscale'
connection_shortDict['nep_connection_family'] = 'nep.family.aws'
connection_shortDict['uuid'] = tmpUUID
connection_shortDict['key'] = outscaleKey
connection_shortDict['secret'] = outscaleSecret
all_objects.append(connection_shortDict)
hasAtLeastOneError=False;
hasAllError=True;
hasOneRegionAllErrors = True;
hasRegionAllErrors = True
isErrorRelevant = False;
regionErrorsStatus = {}
regionErrorGlobalCount = 0
for key in region_mapping_dict:
regionErrorGlobalCount = regionErrorGlobalCount+ 1;
regionErrorsStatus[key] = {'allerror':True, 'oneerror':False, 'errors':[]}
allErrors = []
for key in region_mapping_dict:
for anObj in region_mapping_dict[key]['objects']:
all_objects.append(anObj);
for anObj in region_mapping_dict[key]['conn_status']:
endStatus = region_mapping_dict[key]['conn_status'][anObj]
# pp.pprint(region_mapping_dict[key]['conn_status'])
if endStatus['status'] == 200:
regionErrorsStatus[key]['allerror'] = False
# hasAllError = False;
else:
regionErrorsStatus[key]['oneerror'] = True;
# hasAtLeastOneError = True;
if str(endStatus['status']) not in regionErrorsStatus[key]['errors']:
regionErrorsStatus[key]['errors'].append(str(endStatus['status']))
if regionErrorsStatus[key]['allerror'] == True:
regionErrorsStatus[key]['oneerror'] = False
#For outscale if a region is all false this is due to the authorization problem
# if regionHasAllError == True:
# hasAtLeastOneError = False;
# print "ALLERROR"
connection_dict = dict()
connection_dict['__class_name'] = 'connection'
connection_dict['name'] = outscaleName
connection_dict['nep_connection_type'] = 'nep.type.aws.outscale'
connection_dict['nep_connection_family'] = 'nep.family.aws'
connection_dict['uuid'] = tmpUUID
connection_dict['objects'] = all_objects
connection_dict['key'] = outscaleKey
connection_dict['secret'] = outscaleSecret
hasOneRelevantPartielError = False
regionErrorFinalCount = 0
for key in region_mapping_dict:
if regionErrorsStatus[key]['oneerror'] == True:
isErrorRelevant = True;
if regionErrorsStatus[key]['allerror'] == True:
regionErrorFinalCount = regionErrorFinalCount+1
if isErrorRelevant == False:
if regionErrorFinalCount == regionErrorGlobalCount:
isErrorRelevant = True
hasAllError = True;
if isErrorRelevant:
for key in region_mapping_dict:
for anError in regionErrorsStatus[key]['errors']:
if anError not in allErrors:
allErrors.append(anError)
if isErrorRelevant:
errorDict = {}
errorDict['__class_name'] = 'error'
errorDict['uuid'] = 'error_'+str(uuid.uuid1())
errorDict['codes'] = ','.join(allErrors)
if hasAllError:
errorDict['partial'] = False
errorDict['message'] = 'An error has occured please check that you have proper IAM credentials to access the infrastructure data and that credentials are properly entered'
else:
errorDict['partial'] = True
errorDict['message'] = 'Some errors have occured please check that you have proper IAM credentials to access the infrastructure data.'
errorDict['partial'] = True
connection_dict['error'] = errorDict;
# pp = pprint.PrettyPrinter(indent=4)
# pp.pprint(connection_dict)
return connection_dict
import json
import botocore.session
from botocore.exceptions import ClientError
from scrapy.conf import settings
session = botocore.session.get_session()
session.set_credentials(access_key=settings['AWS_ACCESS_KEY'],
secret_key=settings['AWS_SECRET_KEY'])
client = session.create_client('s3', region_name=settings['S3_REGION'])
def load_members():
try:
response = client.get_object(Bucket=settings['S3_BUCKET'],
Key=settings['S3_KEY'])
members = json.loads(response['Body'].read())
except ClientError:
members = {}
return members
def save_members(members):
client.put_object(ACL='private',
Bucket=settings['S3_BUCKET'],
Key=settings['S3_KEY'],
Body=json.dumps(members))
def main():
#
# parse args
#
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--verbose',
help='verbose output',
action='store_true')
parser.add_argument('command', help='command passed to aws cli', nargs='+')
args = parser.parse_args()
command = ' '.join(args.command)
credentials_file = str(
os.getenv('AWS_ENCRYPTED_CREDENTIALS_FILE', '~/.aws/credentials.gpg'))
config_file = str(os.getenv('AWS_CONFIG_FILE', '~/.aws/config'))
profile = str(
os.getenv('AWS_PROFILE', os.getenv('AWS_DEFAULT_PROFILE', 'default')))
if args.verbose:
print 'AWS_ENCRYPTED_CREDENTIALS_FILE={}'.format(credentials_file)
print 'AWS_CONFIG_FILE={}'.format(config_file)
print 'AWS_DEFAULT_PROFILE={}'.format(profile)
print 'command:', command
#
# parse config
#
config = ConfigParser.ConfigParser()
config.read(os.path.expanduser(config_file))
config_section = 'profile ' + profile
source_profile = profile
role_arn = None
if config.has_option(config_section, 'source_profile'):
source_profile = config.get(config_section, 'source_profile')
if config.has_option(config_section, 'role_arn'):
role_arn = config.get(config_section, 'role_arn')
if args.verbose:
print 'source_profile=' + str(source_profile)
print 'role_arn=' + str(role_arn)
#
# read encrypted file
#
credentials_file = os.path.expanduser(credentials_file)
if not os.path.exists(credentials_file):
sys.exit(
'Unable to find credentials file, {}'.format(credentials_file))
try:
with open(credentials_file, mode='rb') as file:
encrypted = file.read()
except IOError as e:
sys.exit('I/O error({0}): {1}; credentials_file: {2}'.format(
e.errno, e.strerror, credentials_file))
except:
sys.exit('Unexpected error:'.format(sys.exc_info()[0]))
#
# gpgme
#
gme = gpgme.Context()
encrypted_bytes = io.BytesIO(encrypted)
decrypted_bytes = io.BytesIO()
try:
gme.decrypt(encrypted_bytes, decrypted_bytes)
except gpgme.GpgmeError as e:
sys.exit('Unable to decrypt file; {}'.format(e.strerror))
decrypted_string = decrypted_bytes.getvalue().decode('utf8')
#
# parse credentials
#
# make sure to read all of decrypted_bytes
decrypted_bytes.seek(0)
credentials = ConfigParser.ConfigParser()
credentials.readfp(decrypted_bytes)
if not credentials.has_section(source_profile):
sys.exit(
'Credentials file does not have source_profile "{}", AWS_PROFILE={}'
.format(source_profile, profile))
if not credentials.has_option(source_profile, 'aws_access_key_id'):
sys.exit(
'Credentials file source_profile does not have aws_access_key_id, {}'
.format(source_profile))
if not credentials.has_option(source_profile, 'aws_secret_access_key'):
sys.exit(
'Credentials file source_profile does not have aws_secret_access_key, {}'
.format(source_profile))
aws_access_key_id = credentials.get(source_profile, 'aws_access_key_id')
aws_secret_access_key = credentials.get(source_profile,
'aws_secret_access_key')
#
# setup a botocore session
#
session = botocore.session.Session(profile=profile)
session.set_credentials(aws_access_key_id, aws_secret_access_key)
try:
scoped_config = session.get_scoped_config()
region = scoped_config.get('region', None)
if region:
os.putenv('AWS_DEFAULT_REGION', region)
os.putenv('AWS_REGION', region)
except botocore.exceptions.ProfileNotFound as e:
sys.exit('Profile not found in config; profile={}'.format(profile))
if args.verbose:
print 'AWS_DEFAULT_REGION={}'.format(region)
#
# switch iam roles using sts
#
if role_arn == None:
os.putenv('AWS_ACCESS_KEY_ID', aws_access_key_id)
os.putenv('AWS_SECRET_ACCESS_KEY', aws_secret_access_key)
else:
sts_client = session.create_client('sts')
assumed_role_object = sts_client.assume_role(
RoleArn=role_arn, RoleSessionName='AssumeRoleSession1')
role_credentials = assumed_role_object['Credentials']
os.putenv('AWS_ACCESS_KEY_ID', role_credentials['AccessKeyId'])
os.putenv('AWS_SECRET_ACCESS_KEY', role_credentials['SecretAccessKey'])
os.putenv('AWS_SESSION_TOKEN', role_credentials['SessionToken'])
#
# run the command
#
if args.verbose:
print 'Command output:'
my_env = os.environ.copy()
command_status = os.system(command)
exit(os.WEXITSTATUS(command_status))