def build_vpc(aws_creds, region, cidr=""):
print('Building VPC for %s network' % str(cidr))
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
ec2 = session.get_service('ec2')
operation = ec2.get_operation('CreateVpc')
endpoint = ec2.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
CidrBlock=cidr)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
vpc_id = response_data['Vpc']['VpcId']
operation = ec2.get_operation('CreateTags')
http_response, response_data = operation.call(endpoint,
Resources=[vpc_id],
Tags=[{"Key": "Name",
"Value": 'Meetup'
}],
)
return vpc_id
def connect_to_host(cls, host='localhost', port=8000, is_secure=False,
session=None, access_key=None, secret_key=None,
**kwargs):
"""
Connect to a specific host.
This method has been deprecated in favor of :meth:`~.connect`
Parameters
----------
host : str, optional
Address of the host (default 'localhost')
port : int, optional
Connect to the host on this port (default 8000)
is_secure : bool, optional
Enforce https connection (default False)
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
**kwargs : dict
Keyword arguments to pass to the constructor
"""
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
url = "http://%s:%d" % (host, port)
service = session.get_service('dynamodb')
endpoint = service.get_endpoint(
'local', endpoint_url=url, is_secure=is_secure)
return cls(service, endpoint, **kwargs)
def test_xml_parsing():
for dp in ['responses', 'errors']:
data_path = os.path.join(os.path.dirname(__file__), 'xml')
data_path = os.path.join(data_path, dp)
session = botocore.session.get_session()
xml_files = glob.glob('%s/*.xml' % data_path)
service_names = set()
for fn in xml_files:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_xml_files = glob.glob('%s/%s-*.xml' %
(data_path, service_name))
for xmlfile in service_xml_files:
dirname, filename = os.path.split(xmlfile)
basename = os.path.splitext(filename)[0]
jsonfile = os.path.join(dirname, basename + '.json')
sn, opname = basename.split('-', 1)
opname = opname.split('#')[0]
operation = service.get_operation(opname)
r = XmlResponse(session, operation)
fp = open(xmlfile)
xml = fp.read()
fp.close()
r.parse(xml, 'utf-8')
save_jsonfile(jsonfile, r)
fp = open(jsonfile)
data = json.load(fp)
fp.close()
yield check_dicts, xmlfile, r.get_value(), data
def test_json_parsing():
input_path = os.path.join(os.path.dirname(__file__), 'json')
input_path = os.path.join(input_path, 'inputs')
output_path = os.path.join(os.path.dirname(__file__), 'json')
output_path = os.path.join(output_path, 'outputs')
session = botocore.session.get_session()
jsonfiles = glob.glob('%s/*.json' % input_path)
service_names = set()
for fn in jsonfiles:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_input_files = glob.glob('%s/%s-*.json' %
(input_path, service_name))
for inputfile in service_input_files:
dirname, filename = os.path.split(inputfile)
outputfile = os.path.join(output_path, filename)
basename = os.path.splitext(filename)[0]
sn, opname = basename.split('-', 1)
operation = service.get_operation(opname)
r = JSONResponse(session, operation)
with open(inputfile, 'rb') as fp:
jsondoc = fp.read()
r.parse(jsondoc, 'utf-8')
save_jsonfile(outputfile, r)
fp = open(outputfile)
data = json.load(fp)
fp.close()
yield check_dicts, inputfile, r.get_value(), data
def main():
print
print "sending email via ses - initial parameters"
subject = "SES test email"
region = 'us-east-1' #change this to your region
destination = {'ToAddresses': ['*****@*****.**']} # to test ses validate an email address and use that one
source = '*****@*****.**' # this is the from address
msgbody = {'Subject': {'Data': 'this is a subject'}, 'Body': {'Text': {'Data': 'This is the body' + "\nand more lines"}}}
print "setting up the session and sending the email"
session = botocore.session.get_session()
ses = session.get_service('ses')
operation = ses.get_operation('SendEmail')
endpoint = ses.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
source=source,
destination=destination,
message=msgbody)
print "the results returned from the SendEmail operation"
print http_response
print response_data
def test_xml_parsing():
for dp in ['responses', 'errors']:
data_path = os.path.join(os.path.dirname(__file__), 'xml')
data_path = os.path.join(data_path, dp)
session = botocore.session.get_session()
xml_files = glob.glob('%s/*.xml' % data_path)
service_names = set()
for fn in xml_files:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_xml_files = glob.glob('%s/%s-*.xml' % (data_path,
service_name))
for xmlfile in service_xml_files:
dirname, filename = os.path.split(xmlfile)
basename = os.path.splitext(filename)[0]
jsonfile = os.path.join(dirname, basename + '.json')
sn, opname = basename.split('-', 1)
opname = opname.split('#')[0]
operation = service.get_operation(opname)
r = XmlResponse(session, operation)
fp = open(xmlfile)
xml = fp.read()
fp.close()
r.parse(xml, 'utf-8')
save_jsonfile(jsonfile, r)
fp = open(jsonfile)
data = json.load(fp)
fp.close()
yield check_dicts, xmlfile, r.get_value(), data
def retrieve_notifications(self, max_notifications=1):
"""Attempts to retrieve notifications from the provided SQS queue"""
session = botocore.session.get_session()
sqs = session.get_service("sqs")
receive = sqs.get_operation("ReceiveMessage")
http_response, response_data = receive.call(sqs.get_endpoint(self._region),
queue_url=self._queue_url,
wait_time_seconds=20,
max_number_of_messages=max_notifications)
# Swallow up any errors/issues, logging them out
if http_response.status_code != 200 or not "Messages" in response_data:
log.error(u"Failed to retrieve messages from queue %s with status_code %s: %s" %
(self._queue_url, http_response.status_code, response_data))
return []
notifications = []
for msg in response_data.get("Messages", []):
# Construct a message that we can parse into events.
message = Message(self._queue_url, self._region, msg)
try:
notifications.append(ScalingNotification(message))
except Exception:
log.exception(u"Invalid message received; will delete from queue: %s", msg)
message.delete()
return notifications
def connect_to_region(cls, region, session=None, access_key=None,
secret_key=None, **kwargs):
"""
Connect to an AWS region.
This method has been deprecated in favor of :meth:`~.connect`
Parameters
----------
region : str
Name of an AWS region
session : :class:`~botocore.session.Session`, optional
The Session object to use for the connection
access_key : str, optional
If session is None, set this access key when creating the session
secret_key : str, optional
If session is None, set this secret key when creating the session
**kwargs : dict
Keyword arguments to pass to the constructor
"""
if session is None:
session = botocore.session.get_session()
if access_key is not None:
session.set_credentials(access_key, secret_key)
service = session.get_service('dynamodb')
return cls(service, service.get_endpoint(region), **kwargs)
def main():
session = botocore.session.Session()
session.set_credentials(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY)
ec2 = session.get_service('ec2')
endpoint = ec2.get_endpoint(AWS_DEFAULT_REGION)
op_din = ec2.get_operation('DescribeInstances')
op_stop = ec2.get_operation('StopInstances')
http_response, reservation_data = op_din.call(endpoint, filters=[{"Name":"instance-state-name","Values":["running"]}] )
if not http_response.ok:
print(reservation_data['Errors'])
sys.exit(2)
else:
reservations = reservation_data['Reservations']
#Pull the instances out of reservations
instances = []
for reservation in reservations:
instances += reservation['Instances']
print("Found %i instances in running state" % len(instances))
instance_ids = [instance['InstanceId'] for instance in instances]
http_response, data = op_stop.call(endpoint, instance_ids = instance_ids)
if not http_response.ok:
print(data['Errors'])
sys.exit(2)
for instance in data['StoppingInstances']:
print("Instance %s now stopping" % instance['InstanceId'])
def retrieve_events(self, max_events=1):
"""Attempts to retrieve events from the provided SQS queue"""
session = botocore.session.get_session()
sqs = session.get_service("sqs")
receive = sqs.get_operation("ReceiveMessage")
http_response, response_data = receive.call(
sqs.get_endpoint(self._region),
queue_url=self._queue_url,
wait_time_seconds=20,
max_number_of_messages=max_events)
# Swallow up any errors/issues, logging them out
if http_response.status_code != 200 or not "Messages" in response_data:
log.error(
u"Failed to retrieve messages from queue %s with status_code %s: %s"
% (self._queue_url, http_response.status_code, response_data))
return []
events = []
for msg in response_data.get("Messages", []):
# Construct a message that we can parse into events.
message = Message(self._queue_url, self._region, msg)
try:
# Try to parse our message as a custom resource event
event = ResourceEvent(message)
events.append(event)
except Exception:
log.exception(
u"Invalid message received; will delete from queue: %s",
msg)
message.delete()
return events
def test_response_parsing():
data_path = os.path.join(os.path.dirname(__file__), 'data')
session = botocore.session.get_session()
xml_files = glob.glob('%s/*.xml' % data_path)
service_names = set()
for fn in xml_files:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_xml_files = glob.glob('%s/%s-*.xml' % (data_path, service_name))
for xmlfile in service_xml_files:
dirname, filename = os.path.split(xmlfile)
basename = os.path.splitext(filename)[0]
jsonfile = os.path.join(dirname, basename + '.json')
sn, opname = basename.split('-', 1)
operation = service.get_operation(opname)
r = XmlResponse(operation)
fp = open(xmlfile)
xml = fp.read()
fp.close()
r.parse(xml)
# This is a little convenience when creating new tests.
# You just have to drop the XML file into the data directory
# and then, if not JSON file is present, this code will
# create the JSON file and dump the parsed value into it.
# Of course, you need to validate that the JSON is correct
# but it makes it easy to bootstrap more tests.
if not os.path.isfile(jsonfile):
fp = open(jsonfile, 'w')
json.dump(r.get_value(), fp, indent=4)
fp.close()
fp = open(jsonfile)
data = json.load(fp)
fp.close()
yield check_dicts, xmlfile, r.get_value(), data
def test_json_parsing():
input_path = os.path.join(os.path.dirname(__file__), 'json')
input_path = os.path.join(input_path, 'inputs')
output_path = os.path.join(os.path.dirname(__file__), 'json')
output_path = os.path.join(output_path, 'outputs')
session = botocore.session.get_session()
jsonfiles = glob.glob('%s/*.json' % input_path)
service_names = set()
for fn in jsonfiles:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_input_files = glob.glob('%s/%s-*.json' % (input_path,
service_name))
for inputfile in service_input_files:
dirname, filename = os.path.split(inputfile)
outputfile = os.path.join(output_path, filename)
basename = os.path.splitext(filename)[0]
sn, opname = basename.split('-', 1)
operation = service.get_operation(opname)
r = JSONResponse(session, operation)
with open(inputfile, 'rb') as fp:
jsondoc = fp.read()
r.parse(jsondoc, 'utf-8')
save_jsonfile(outputfile, r)
fp = open(outputfile)
data = json.load(fp)
fp.close()
yield check_dicts, inputfile, r.get_value(), data
def build_elb(aws_creds, region, lb_name="", subnets=[], elb_sg_id=""):
print('Building elb')
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
elb = session.get_service('elb')
operation = elb.get_operation('CreateLoadBalancer')
endpoint = elb.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
LoadBalancerName=lb_name,
Listeners=[
{"Protocol": 'http',
"LoadBalancerPort": 80,
"InstanceProtocol": 'http',
"InstancePort": 80}],
Subnets=subnets,
SecurityGroups=[elb_sg_id],
Scheme='internal')
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
elb_name = response_data['DNSName']
return elb_name
def main():
print "some dynamoDb examples"
session = botocore.session.get_session()
db = session.get_service('dynamodb')
print "DescribeTable"
operation = db.get_operation('DescribeTable')
endpoint = db.get_endpoint('us-east-1') # change this to your region
tableName = 'myTable' #change this to your dynamoDb test table name
http_response, response_data = operation.call(endpoint,
table_name=tableName)
print
print "results of DescribeTable for tableName:" + tableName
print http_response
print response_data
getops = db.get_operation('GetItem')
# assumes a numeric hash key as the primaryKey and named customerId
# you can create this item in the aws console if you want
getItem = {u'customerId':{"N":"1"}}
http_response, response_data = getops.call(endpoint, table_name='myTable', key=getItem)
print
print http_response
print "getItem Results: " + str(response_data)
print
print "Insert a few hundred Items (records) in " + tableName
putops = db.get_operation('PutItem')
i = 0
x = {}
y = {}
while i < 200:
print i
string = "this is a made up key string for itemId=" + str(i)
putItem = {u'customerId':{"N":i}}
x['N']=str(i)
y['S']=string
putItem[u'customerId']=x
putItem[u'astringattribute']=y
# 'N' tells AWS that the string should be treated as a number
# 'S' treat the string as a string
# 'SS' treat the string as a set of strings (like an array)
putItem[u'setstringattribute'] = {u'SS':(u"x", u"y")}
# uncomment out the print putItem if you want to see what we are inserting into DynamoDb
#print putItem
http_response, response_data = putops.call(endpoint, table_name='myTable', item=putItem)
i = i + 1
# dynamoDb throttles inserts so you may see them slow down in your terminal window
# lets put a little delay into the inserts ourselves
if i % 10 == 0:
time.sleep(2)
def init_botocore_service(name, region):
"""
Initialize the proper service with botocore
"""
session = botocore.session.get_session()
service = session.get_service(name)
endpoint = service.get_endpoint(region)
return service, endpoint
def init_botocore_service(name, region):
"""
Initialize the proper service with botocore
"""
session = botocore.session.get_session()
service = session.get_service(name)
endpoint = service.get_endpoint(region)
return service, endpoint
def test_get_endpoint_with_no_region(self):
self.environ["AWS_ACCESS_KEY_ID"] = "access_key"
self.environ["AWS_SECRET_ACCESS_KEY"] = "secret_key"
session = botocore.session.get_session()
# Test global endpoint service such as iam.
service = session.get_service("iam")
endpoint = service.get_endpoint()
self.assertEqual(endpoint.host, "https://iam.amazonaws.com/")
def test_get_session_token(self):
session = botocore.session.get_session()
sns = session.get_service('sts')
op = sns.get_operation('GetSessionToken')
params = {}
endpoint = self.get_mocked_endpoint()
with self.assertRaises(NoCredentialsError):
endpoint.make_request(op, params)
def test_can_make_request():
session = botocore.session.get_session()
for service_name in SMOKE_TESTS:
service = session.get_service(service_name)
endpoint = service.get_endpoint(REGION)
for operation_name in SMOKE_TESTS[service_name]:
kwargs = SMOKE_TESTS[service_name][operation_name]
yield _make_call, service, endpoint, operation_name, kwargs
def test_get_session_token(self):
session = botocore.session.get_session()
sns = session.get_service('sts')
op = sns.get_operation('GetSessionToken')
params = {}
endpoint = self.get_mocked_endpoint()
with self.assertRaises(NoCredentialsError):
endpoint.make_request(op, params)
def test_get_session_token(self):
with self.get_credentials_patch as mock_fn:
session = botocore.session.get_session()
sns = session.get_service("sts")
op = sns.get_operation("GetSessionToken")
params = {}
endpoint = self.get_mocked_endpoint()
self.assertRaises(NoCredentialsError, endpoint.make_request, op, params)
def test_can_make_request():
session = botocore.session.get_session()
for service_name in SMOKE_TESTS:
service = session.get_service(service_name)
endpoint = service.get_endpoint(REGION)
for operation_name in SMOKE_TESTS[service_name]:
kwargs = SMOKE_TESTS[service_name][operation_name]
yield _make_call, service, endpoint, operation_name, kwargs
def put_object(self, bucket, key, content):
session = botocore.session.get_session()
service = session.get_service("s3")
endpoint = service.get_endpoint("us-east-1")
http, response = service.get_operation("CreateBucket").call(endpoint, bucket=bucket)
time.sleep(5)
self.addCleanup(service.get_operation("DeleteBucket").call, endpoint, bucket=bucket)
http, response = service.get_operation("PutObject").call(endpoint, bucket=bucket, key=key, body=content)
self.addCleanup(service.get_operation("DeleteObject").call, endpoint, bucket=bucket, key=key)
def gen_s3_obj(method):
config = common.get_config()
session = botocore.session.get_session()
session.set_credentials(config['access_key'], config['secret_key'])
s3 = session.get_service('s3')
operation = s3.get_operation(method)
endpoint = s3.get_endpoint(config['region'])
return [endpoint, operation]
def gen_s3_obj(method):
config = common.get_config()
session = botocore.session.get_session()
session.set_credentials(config["access_key"], config["secret_key"])
s3 = session.get_service("s3")
operation = s3.get_operation(method)
endpoint = s3.get_endpoint(config["region"])
return [endpoint, operation]
def test_assume_role_with_saml(self):
session = botocore.session.get_session()
sns = session.get_service('sts')
op = sns.get_operation('AssumeRoleWithSAML')
self.assertEqual(op.signature_version, None)
endpoint = self.get_mocked_endpoint()
params = op.build_parameters(principal_arn='principal_arn',
role_arn='role_arn',
saml_assertion='saml_assertion')
endpoint.make_request(op, params)
self.assertNotIn('auth', self.called_params)
self.reset_called_params()
def main():
print
print "Putting a file in s3 - initial parameters"
bucket = 'yourbucketname' # change this to your bucket name
key = 'xyzzy' # a subfolder under the bucket
filename ='testfile.txt' # the file we will put into S3
region = 'us-east-1' # change this to your region
acl = 'public-read' # we are going to set the ACL to public-read so we can access the file via a url
print
print ' region: ' + region
print ' bucket: ' + bucket
print 'key (subfolder): ' + key
print ' filename: ' + './' + filename
print ' acl: ' + acl
session = botocore.session.get_session()
s3 = session.get_service('s3')
operation = s3.get_operation('PutObject')
endpoint = s3.get_endpoint(region)
print
print "uploading the file to s3"
fp = open('./' + filename, 'rb')
http_response, response_data = operation.call(endpoint,
bucket=bucket,
key=key + '/' + filename,
body=fp)
print http_response
print response_data
print
print "getting s3 object properties of file we just uploaded"
operation = s3.get_operation('GetObjectAcl')
http_response, response_data = operation.call(endpoint,
bucket=bucket,
key=key + '/' + filename)
print http_response
print response_data
print
print "setting the acl to public-read"
operation = s3.get_operation('PutObjectAcl')
http_response, response_data = operation.call(endpoint,
bucket=bucket,
key=key + '/' + filename,
acl=acl)
print http_response
print response_data
print
print "The url of the object is:"
print
print 'http://'+bucket+'.s3.amazonaws.com/'+ key + '/' + filename
def test_assume_role_with_saml(self):
session = botocore.session.get_session()
sns = session.get_service('sts')
op = sns.get_operation('AssumeRoleWithSAML')
self.assertEqual(op.signature_version, None)
endpoint = self.get_mocked_endpoint()
params = op.build_parameters(principal_arn='principal_arn',
role_arn='role_arn',
saml_assertion='saml_assertion')
endpoint.make_request(op, params)
self.assertNotIn('auth', self.called_params)
self.reset_called_params()
def put_object(self, bucket, key, content):
session = botocore.session.get_session()
service = session.get_service('s3')
endpoint = service.get_endpoint('us-east-1')
http, response = service.get_operation(
'CreateBucket').call(endpoint, bucket=bucket)
time.sleep(5)
self.addCleanup(service.get_operation('DeleteBucket').call,
endpoint, bucket=bucket)
http, response = service.get_operation('PutObject').call(
endpoint, bucket=bucket, key=key, body=content)
self.addCleanup(service.get_operation('DeleteObject').call,
endpoint, bucket=bucket, key=key)
def test_assume_role_with_saml(self):
with self.get_credentials_patch as mock_fn:
session = botocore.session.get_session()
sns = session.get_service("sts")
op = sns.get_operation("AssumeRoleWithSAML")
self.assertEqual(op.signature_version, None)
endpoint = self.get_mocked_endpoint()
params = op.build_parameters(
principal_arn="principal_arn", role_arn="role_arn", saml_assertion="saml_assertion"
)
endpoint.make_request(op, params)
self.assertNotIn("auth", self.called_params)
self.reset_called_params()
def resource_from_arn(arn, data):
session = botocore.session.get_session()
parts = ArnComponents(*arn.split(':', 6))
service = session.get_service(parts.service)
if ':' in parts.resource:
resource_type, resource_id = parts.resource.split(':')
elif '/' in parts.resource:
resource_type, resource_id = parts.resource.split('/')
else:
resource_type = parts.resource
endpoint = Endpoint(service, parts.region, parts.account)
resource_path = '.'.join(['aws', parts.service, resource_type])
resource_cls = find_resource_class(resource_path)
return resource_cls(endpoint, data)
def resource_from_arn(arn, data):
session = botocore.session.get_session()
parts = ArnComponents(*arn.split(':', 6))
service = session.get_service(parts.service)
if ':' in parts.resource:
resource_type, resource_id = parts.resource.split(':')
elif '/' in parts.resource:
resource_type, resource_id = parts.resource.split('/')
else:
resource_type = parts.resource
endpoint = Endpoint(service, parts.region, parts.account)
resource_path = '.'.join(['aws', parts.service, resource_type])
resource_cls = find_resource_class(resource_path)
return resource_cls(endpoint, data)
def push_file_to_s3(file):
#print("Putting a file in s3 - initial parameters")
bucket = IMS.settings.BUCKET_NAME # change this to your bucket name
key = 'croomer_production' # a subfolder under the bucket
filelocation = IMS.settings.MEDIA_ROOT + '/' + file.name # the file we will put into S3
filename = file.name
region = 'us-west-2' # change this to your region
acl = 'public-read' # we are going to set the ACL to public-read so we can access the file via a url
#print( ' region: ' + region)
#print( ' bucket: ' + bucket)
#print( 'key (subfolder): ' + key)
#print( ' filename: ' + './' + filename)
#print( ' acl: ' + acl)
session = botocore.session.get_session()
s3 = session.get_service('s3')
operation = s3.get_operation('PutObject')
endpoint = s3.get_endpoint(region)
#print("uploading the file to s3")
fp = open('./' + filelocation, 'rb')
http_response, response_data = operation.call(endpoint,
bucket=bucket,
key=key + '/' + filename,
body=fp)
#print (http_response)
#print (response_data)
#print("getting s3 object properties of file we just uploaded")
operation = s3.get_operation('GetObjectAcl')
http_response, response_data = operation.call(endpoint,
bucket=bucket,
key=key + '/' + filename)
#print( http_response)
#print( response_data)
#print("setting the acl to public-read")
operation = s3.get_operation('PutObjectAcl')
http_response, response_data = operation.call(endpoint,
bucket=bucket,
key=key + '/' + filename,
acl=acl)
#print( http_response)
#print( response_data)
#print("The url of the object is:")
#print ('http://'+bucket+'.s3.amazonaws.com/'+ key + '/' + filename)
return 'http://'+bucket+'.s3.amazonaws.com/'+ key + '/' + filename
def _toggle_term_protect(name, enabled):
'''
Toggle termination protection on a node
'''
# region is required for all boto queries
region = get_location(None)
# init botocore
session = botocore.session.get_session()
session.set_credentials(
access_key=__opts__['AWS.id'],
secret_key=__opts__['AWS.key'],
)
service = session.get_service('ec2')
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {
'instance_id': node.id,
'attribute': 'disableApiTermination',
'value': 'true' if enabled else 'false',
}
# get instance information
operation = service.get_operation('modify-instance-attribute')
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = (
'Termination protection successfully {0} on {1}'.format(
enabled and 'enabled' or 'disabled',
name
)
)
log.info(msg)
return msg
else:
msg = (
'Bad response from AWS: {0}'.format(
http_response.status_code
)
)
log.error(msg)
return msg
def _toggle_term_protect(name, enabled):
'''
Toggle termination protection on a node
'''
# region is required for all boto queries
region = get_location(None)
# init botocore
vm_ = get_configured_provider()
session = botocore.session.get_session() # pylint: disable=E0602
session.set_credentials(
access_key=config.get_cloud_config_value(
'id', vm_, __opts__, search_global=False
),
secret_key=config.get_cloud_config_value(
'key', vm_, __opts__, search_global=False
)
)
service = session.get_service('ec2')
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {
'instance_id': node.id,
'attribute': 'disableApiTermination',
'value': 'true' if enabled else 'false',
}
# get instance information
operation = service.get_operation('modify-instance-attribute')
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = 'Termination protection successfully {0} on {1}'.format(
enabled and 'enabled' or 'disabled',
name
)
log.info(msg)
return msg
# No proper HTTP response!?
msg = 'Bad response from AWS: {0}'.format(http_response.status_code)
log.error(msg)
return msg
def _toggle_term_protect(name, enabled):
'''
Toggle termination protection on a node
'''
# region is required for all boto queries
region = get_location(None)
# init botocore
vm_ = get_configured_provider()
session = botocore.session.get_session() # pylint: disable=E0602
session.set_credentials(
access_key=config.get_cloud_config_value(
'id', vm_, __opts__, search_global=False
),
secret_key=config.get_cloud_config_value(
'key', vm_, __opts__, search_global=False
)
)
service = session.get_service('ec2')
endpoint = service.get_endpoint(region)
# get the instance-id for the supplied node name
conn = get_conn(location=region)
node = get_node(conn, name)
params = {
'instance_id': node.id,
'attribute': 'disableApiTermination',
'value': 'true' if enabled else 'false',
}
# get instance information
operation = service.get_operation('modify-instance-attribute')
http_response, response_data = operation.call(endpoint, **params)
if http_response.status_code == 200:
msg = 'Termination protection successfully {0} on {1}'.format(
enabled and 'enabled' or 'disabled',
name
)
log.info(msg)
return msg
# No proper HTTP response!?
msg = 'Bad response from AWS: {0}'.format(http_response.status_code)
log.error(msg)
return msg
def destroy_subnet(aws_creds, region, subnet_id=""):
print('Destroying subnet %s' % str(subnet_id))
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
ec2 = session.get_service('ec2')
operation = ec2.get_operation('DeleteSubnet')
endpoint = ec2.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
SubnetId=subnet_id)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
return True
def destroy_elb(aws_creds, region, lb_name=""):
print('Destroying elb')
session = botocore.session.get_session()
session.set_credentials(aws_creds.access_key, aws_creds.secret_key)
elb = session.get_service('elb')
operation = elb.get_operation('DeleteLoadBalancer')
endpoint = elb.get_endpoint(region)
http_response, response_data = operation.call(endpoint,
LoadBalancerName=lb_name)
p3.pprint(str(http_response.status_code) + " - " + http_response.reason)
p3.pprint(response_data)
if http_response.status_code != 200:
raise(ApiException)
return True
def main():
start = time.time()
print
print "Show how to use the Batch function with SQS queues"
# change this to your queue_url
queue_url = 'https://some queue url'
region = 'us-east-1' # change this to your region
print 'queue_url: ' + queue_url
print ' region: ' + region
session = botocore.session.get_session()
sqs = session.get_service('sqs')
endpoint = sqs.get_endpoint(region)
send = sqs.get_operation('SendMessageBatch')
print "Send 99 messages, 10 at a time"
# message bodies can be json, xml, plan text and a lot more
message_body_prefix = "1234567890_" # this can be any ascii text and can contain some other characters
entries = list()
counter=0
while counter <99:
# send batch every 10 messages,
entry = {'Id': "testmsg_" + str(counter), 'MessageBody' : message_body_prefix + str(counter)}
entries.append(entry)
# bacth size of up to 10 messages allowed
if (counter+1) % 10 == 0:
http_response, response_data = send.call(endpoint, queue_url = queue_url, entries = entries)
# in production you can get a 200 status value and some of the messages still might have failed to be sent
# see the aws documentation on how to detect individual message failures
if not http_response.status_code == 200:
print "error sending batch messages: " + str(http_response.status_code) + " " + str(response_data)
entries = list()
counter += 1
# process any remaining messages
if len(entries) > 0:
http_response, response_data = send.call(endpoint, queue_url = queue_url, entries = entries)
if not http_response.status_code == 200:
print "error sending batch messages: " + str(http_response.status_code) + " " + str(response_data)
print "Total elapsed time (sec): " + str(time.time() - start)
def find_instance(instance_id, config_profiles):
"""
Search through all AWS profiles and regions for an instance.
"""
profiles_session = botocore.session.get_session()
for profile in profiles_session.available_profiles:
# Re-using the same session doesn't work
session = botocore.session.get_session()
session.profile = profile
# Prefer regions listed in the profile
regions = None
if profile in config_profiles:
regions = config_profiles[profile].regions
if not regions:
region = session.get_config_variable('region')
if not region:
continue
else:
regions = [region]
ec2 = session.get_service('ec2')
operation = ec2.get_operation('DescribeInstances')
for region in regions:
logger.debug("connecting to region '%s' with AWS profile '%s'...",
region, profile)
endpoint = ec2.get_endpoint(region)
try:
resp, data = operation.call(
endpoint,
instance_ids=[instance_id],
)
except botocore.exceptions.NoCredentialsError:
break
if resp.status_code == 200:
for reservation in data['Reservations']:
for instance_data in reservation['Instances']:
return Instance.from_instance_data(instance_data,
profile, region)
else:
continue
return None
def do_action(region, awsKey, awsSecret, ownerID, actionObject, actionName, objectUUID, parameters):
session = botocore.session.get_session()
session.set_credentials(outscaleKey, outscaleSecret)
ec2 = session.get_service('ec2')
if actionObject == 'instance':
if actionName == 'start':
try:
# connEC2.start_instances(instance_ids=[objectUUID])
operation = ec2.get_operation('StartInstances');
http_response, response_data = operation.call(endpoint,instance_ids = [objectUUID])
return 200, ""
except boto.exception.EC2ResponseError as e:
description = "Unknown description"
if len(e.errors):
tmpVar = e.errors[0]
if len(tmpVar):
description = tmpVar[0]
return e.status, description
elif actionName == 'stop':
try:
operation = ec2.get_operation('StopInstances');
http_response, response_data = operation.call(endpoint,instance_ids = [objectUUID])
return 200, ""
except boto.exception.EC2ResponseError as e:
description = "Unknown description"
if len(e.errors):
tmpVar = e.errors[0]
if len(tmpVar):
description = tmpVar[0]
return e.status, description
elif actionObject == 'volume':
if actionName == 'delete':
try:
operation = ec2.get_operation('DeleteVolume');
http_response, response_data = operation.call(endpoint,volume_ids = [objectUUID])
return 200,""
except boto.exception.EC2ResponseError as e:
description = "Unknown description"
if len(e.errors):
tmpVar = e.errors[0]
if len(tmpVar):
description = tmpVar[0]
return e.status, description
pass;
def test_xml_parsing():
for dp in ['responses', 'errors']:
data_path = os.path.join(os.path.dirname(__file__), 'xml')
data_path = os.path.join(data_path, dp)
session = create_session()
xml_files = glob.glob('%s/*.xml' % data_path)
service_names = set()
for fn in xml_files:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_xml_files = glob.glob('%s/%s-*.xml' %
(data_path, service_name))
for xmlfile in service_xml_files:
expected = _get_expected_parsed_result(xmlfile)
operation_obj = _get_operation_obj(service, xmlfile)
raw_response_body = _get_raw_response_body(xmlfile)
yield _test_parsed_response, xmlfile, raw_response_body, operation_obj, expected
def _make_call_with_errors(session, service_name, region_name, operation_name,
kwargs):
service = session.get_service(service_name)
endpoint = service.get_endpoint(region_name)
operation = service.get_operation(operation_name)
original_send = adapters.HTTPAdapter.send
def mock_http_adapter_send(self, *args, **kwargs):
if not getattr(self, '_integ_test_error_raised', False):
self._integ_test_error_raised = True
raise ConnectionError("Simulated ConnectionError raised.")
else:
return original_send(self, *args, **kwargs)
with mock.patch('botocore.vendored.requests.adapters.HTTPAdapter.send',
mock_http_adapter_send):
response = operation.call(endpoint, **kwargs)[1]
assert_true(
'Error' not in response, "Request was not retried properly, "
"received error:\n%s" % pformat(response))
def launch_cmd(role_arn, profile=None):
session = botocore.session.get_session()
session.profile = profile
sts = session.get_service('sts')
endpoint = sts.get_endpoint()
op = sts.get_operation('AssumeRole')
creds = op.call(endpoint, role_arn=role_arn, role_session_name='foobar')[1]
d = {'sessionId': creds['Credentials']['AccessKeyId'],
'sessionKey': creds['Credentials']['SecretAccessKey'],
'sessionToken': creds['Credentials']['SessionToken']}
json_str = json.dumps(d)
params = {'Action': 'getSigninToken',
'Session': json_str}
r = botocore.vendored.requests.get(sign_in_url, params=params)
d = json.loads(r.text)
d['Action'] = 'login'
d['Issuer'] = issuer_url
d['Destination'] = console_url
uri = sign_in_url + '?' + urlencode(d)
webbrowser.open(uri)
def _uhg_test_json_parsing():
input_path = os.path.join(os.path.dirname(__file__), 'json')
input_path = os.path.join(input_path, 'inputs')
output_path = os.path.join(os.path.dirname(__file__), 'json')
output_path = os.path.join(output_path, 'outputs')
session = botocore.session.get_session()
jsonfiles = glob.glob('%s/*.json' % input_path)
service_names = set()
for fn in jsonfiles:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_json_files = glob.glob('%s/%s-*.json' %
(input_path, service_name))
for jsonfile in service_json_files:
expected = _get_expected_parsed_result(jsonfile)
operation_obj = _get_operation_obj(service, jsonfile)
with open(jsonfile, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, jsonfile, raw_response_body, operation_obj, expected
def test_json_parsing():
input_path = os.path.join(os.path.dirname(__file__), 'json')
input_path = os.path.join(input_path, 'inputs')
output_path = os.path.join(os.path.dirname(__file__), 'json')
output_path = os.path.join(output_path, 'outputs')
session = botocore.session.get_session()
jsonfiles = glob.glob('%s/*.json' % input_path)
service_names = set()
for fn in jsonfiles:
service_names.add(os.path.split(fn)[1].split('-')[0])
for service_name in service_names:
service = session.get_service(service_name)
service_input_files = glob.glob('%s/%s-*.json' %
(input_path, service_name))
for inputfile in service_input_files:
dirname, filename = os.path.split(inputfile)
outputfile = os.path.join(output_path, filename)
basename = os.path.splitext(filename)[0]
sn, opname = basename.split('-', 1)
operation = service.get_operation(opname)
r = JSONResponse(session, operation)
headers = {}
with open(inputfile, 'r') as fp:
jsondoc = fp.read()
# Try to get any headers using a special key
try:
parsed = json.loads(jsondoc)
except ValueError:
# This will error later, let it go on
parsed = {}
if '__headers' in parsed:
headers = parsed['__headers']
del parsed['__headers']
jsondoc = json.dumps(parsed)
r.parse(jsondoc.encode('utf-8'), 'utf-8')
r.merge_header_values(headers)
save_jsonfile(outputfile, r)
fp = open(outputfile)
data = json.load(fp)
fp.close()
yield check_dicts, inputfile, r.get_value(), data
def _get_instance_ip(self, instance_id):
session = botocore.session.get_session()
ec2 = session.get_service("ec2")
describe = ec2.get_operation("DescribeInstances")
response, data = describe.call(ec2.get_endpoint(
self._notification.region),
instance_ids=[instance_id])
# Swallow up any error responses and return nothing.
if response.status_code != 200 or 'Reservations' not in data:
log.error(
u"Failed to describe instance %s with status_code %s: %s" %
(instance_id, response.status_code, data))
return None
for reservation in data.get('Reservations', []):
for instance in reservation.get('Instances', []):
if instance.get('InstanceId', '') == instance_id:
return instance.get('PublicIpAddress')
return None
def test_json_errors_parsing():
# The outputs/ directory has sample output responses
# For each file in outputs/ there's a corresponding file
# in expected/ that has the expected parsed response.
base_dir = os.path.join(os.path.dirname(__file__), 'json')
json_responses_dir = os.path.join(base_dir, 'errors')
expected_parsed_dir = os.path.join(base_dir, 'expected')
session = botocore.session.get_session()
for json_response_file in os.listdir(json_responses_dir):
# Files look like: 'datapipeline-create-pipeline.json'
service_name, operation_name = os.path.splitext(
json_response_file)[0].split('-', 1)
expected_parsed_response = os.path.join(expected_parsed_dir,
json_response_file)
raw_response_file = os.path.join(json_responses_dir,
json_response_file)
with open(expected_parsed_response) as f:
expected = json.load(f)
service_obj = session.get_service(service_name)
operation_obj = service_obj.get_operation(operation_name)
with open(raw_response_file, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, raw_response_file, raw_response_body, operation_obj, expected
def get_processor_id(self):
session = botocore.session.get_session()
autoscaling = session.get_service("autoscaling")
describe = autoscaling.get_operation("DescribeTags")
response, data = describe.call(
autoscaling.get_endpoint(self._notification.region),
filters=[{
"Name": "auto-scaling-group",
"Values": [self._notification.auto_scaling_group]
}])
# Swallow up any error responses and return nothing.
if response.status_code != 200 or 'Tags' not in data:
log.error(
u"Failed to retrieve tags for ASG %s with status_code %s: %s" %
(self._notification.auto_scaling_group, response.status_code,
data))
return None
for tag in data.get('Tags', []):
if tag.get('Key', '') == 'ProcessorId':
return tag.get('Value')
return None
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
# http://aws.amazon.com/apache2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
#
import botocore.session
session = botocore.session.get_session()
service = session.get_service('opsworks')
endpoint = service.get_endpoint('us-east-1')
def _make_api_call(api_operation, **kwargs):
"""
Make an API call using botocore for the given api operation.
:param api_operation: Operation name to perform
:param kwargs: Any additional arguments to be passed to the service call
:return: If an OK response returned, returns the data from the call. Will exit(1) otherwise
"""
operation = service.get_operation(api_operation)
response, response_data = operation.call(endpoint, **kwargs)
if response.ok:
return response_data
def collect(model, hours):
row = model.progress.get_item(name='end_time')
if row['timestamp'] is None:
logging.debug('using initial window of -%d hours', hours)
start_time = arrow.utcnow().replace(hours=-hours)
else:
start_time = arrow.get(row['timestamp'])
logging.debug('start time: %s', start_time)
end_time = arrow.utcnow()
logging.debug('end time: %s', end_time)
all_regions = set()
all_product_descriptions = set()
all_instance_types = set()
all_instance_zones = set()
session = botocore.session.get_session()
ec2 = session.get_service('ec2')
operation = ec2.get_operation('DescribeSpotPriceHistory')
for region in ec2.region_names:
if any(region.startswith(x) for x in _EXCLUDED_REGION_PREFIXES):
continue
all_regions.add(region)
next_token = None
while True:
logging.debug('collecting spot prices from region: %s', region)
endpoint = ec2.get_endpoint(region)
if next_token:
response, data = operation.call(
endpoint,
start_time=start_time.format(_FMT),
end_time=end_time.format(_FMT),
next_token=next_token,
)
else:
response, data = operation.call(
endpoint,
start_time=start_time.format(_FMT),
)
next_token = data.get('NextToken')
logging.debug('next_token: %s', next_token)
spot_data = data.get('SpotPriceHistory', [])
#conn = boto.ec2.connect_to_region(r.name)
#logging.debug('getting spot prices for region: %s', r.name)
#data = conn.get_spot_price_history(start_time=start_time)
logging.debug('saving %d spot prices for region: %s',
len(spot_data), region)
with model.spot_prices.batch_write() as batch:
for d in spot_data:
all_product_descriptions.add(d['ProductDescription'])
all_instance_types.add(d['InstanceType'])
all_instance_zones.add((
d['ProductDescription'],
d['InstanceType'],
d['AvailabilityZone'],
))
batch.put_item(
data={
'instance_zone_id':
':'.join([
d['ProductDescription'],
d['InstanceType'],
d['AvailabilityZone'],
]),
'timestamp':
arrow.get(d['Timestamp']).timestamp,
'price':
decimal.Decimal(str(d['SpotPrice'])),
})
if not next_token:
break
logging.debug('saving %d regions', len(all_regions))
with model.regions.batch_write() as batch:
for i in all_regions:
batch.put_item(data={'region': i})
logging.debug('saving %d product_descriptions',
len(all_product_descriptions))
with model.product_descriptions.batch_write() as batch:
for i in all_product_descriptions:
batch.put_item(data={'product_description': i})
logging.debug('saving %d instance_types', len(all_instance_types))
with model.instance_types.batch_write() as batch:
for i in all_instance_types:
batch.put_item(data={'instance_type': i})
logging.debug('saving %d instance_zones', len(all_instance_zones))
with model.instance_zones.batch_write() as batch:
for i in all_instance_zones:
batch.put_item(data={
'instance_id': ':'.join([i[0], i[1]]),
'zone': i[2],
})
logging.debug('saving end_time')
with model.progress.batch_write() as batch:
batch.put_item(data={
'name': 'end_time',
'timestamp': end_time.timestamp,
})
def saml_login(profile,
region,
url,
user,
password=None,
role=None,
print_env_vars=False,
overwrite_default_credentials=False):
session = requests.Session()
response = session.get(url)
keyring_key = 'aws-minion.saml'
password = password or keyring.get_password(keyring_key, user)
if not password:
password = click.prompt('Password', hide_input=True)
with Action('Authenticating against {url}..', **vars()) as act:
# NOTE: parameters are hardcoded for Shibboleth IDP
data = {'j_username': user, 'j_password': password, 'submit': 'Login'}
response2 = session.post(response.url, data=data)
saml_xml = get_saml_response(response2.text)
if not saml_xml:
act.error('LOGIN FAILED')
click.secho(
'SAML login with user "{}" failed, please check your username and password.\n'
.format(user) +
'You might need to change the password in your keyring (e.g. Mac OS X keychain) '
+ 'or use the "--password" option.',
bold=True,
fg='blue')
return
url = get_form_action(response2.text)
encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64')
response3 = session.post(url, data={'SAMLResponse': encoded_xml})
account_names = get_account_names(response3.text)
keyring.set_password(keyring_key, user, password)
with Action('Checking SAML roles..') as act:
roles = get_roles(saml_xml)
if not roles:
act.error('NO VALID ROLE FOUND')
return
if len(roles) == 1:
provider_arn, role_arn = roles[0]
elif role:
matching_roles = [
_role for _role in roles
if role in get_role_label(_role, account_names)
]
if not matching_roles or len(matching_roles) > 1:
raise click.UsageError(
'Given role (--role) was not found or not unique')
provider_arn, role_arn = matching_roles[0]
else:
roles.sort()
provider_arn, role_arn = choice(
'Multiple roles found, please select one.',
[(r, get_role_label(r, account_names)) for r in roles])
with Action('Assuming role "{role_label}"..',
role_label=get_role_label((provider_arn, role_arn),
account_names)):
saml_assertion = codecs.encode(saml_xml.encode('utf-8'),
'base64').decode('ascii').replace(
'\n', '')
# botocore NEEDS some credentials, but does not care about their actual values
os.environ['AWS_ACCESS_KEY_ID'] = 'fake123'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123'
try:
session = botocore.session.get_session()
sts = session.get_service('sts')
operation = sts.get_operation('AssumeRoleWithSAML')
endpoint = sts.get_endpoint(region)
endpoint._signature_version = None
http_response, response_data = operation.call(
endpoint,
role_arn=role_arn,
principal_arn=provider_arn,
SAMLAssertion=saml_assertion)
finally:
del os.environ['AWS_ACCESS_KEY_ID']
del os.environ['AWS_SECRET_ACCESS_KEY']
key_id = response_data['Credentials']['AccessKeyId']
secret = response_data['Credentials']['SecretAccessKey']
session_token = response_data['Credentials']['SessionToken']
if print_env_vars:
# different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both
click.secho(dedent('''\
# environment variables with temporary AWS credentials:
export AWS_ACCESS_KEY_ID="{key_id}"
export AWS_SECRET_ACCESS_KEY="{secret}"
export AWS_SESSION_TOKEN="{session_token}")
export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()),
fg='blue')
profiles_to_write = set([profile])
if overwrite_default_credentials:
profiles_to_write.add('default')
with Action('Writing temporary AWS credentials..'):
for prof in profiles_to_write:
write_aws_credentials(prof, key_id, secret, session_token)
def main(st, et):
if st:
start_time = st
else:
start_time = arrow.utcnow().replace(minutes=common.DEFAULT_LOOKBACK_MINUTES)
if et:
end_time = et
else:
end_time = arrow.utcnow()
all_regions = set()
all_product_descriptions = set()
all_instance_types = set()
all_instance_zones = set()
session = botocore.session.get_session()
ec2 = session.get_service('ec2')
operation = ec2.get_operation('DescribeSpotPriceHistory')
local_timeseries = {}
vals = {}
tss = {}
print 'Preparing...'
for region in AWS_ON_DEMAND_PRICES:
reg_key = region.replace('-','_')
if region not in vals:
vals[reg_key] = {}
tss[reg_key] = {}
for zone in AWS_REGIONS_TO_ZONES[region]:
# print 'Zone: %s' % zone
if zone not in vals[reg_key]:
vals[reg_key][zone] = {}
tss[reg_key][zone] = {}
for product in AWS_ON_DEMAND_PRICES[region]:
# print 'Product: %s' % product
if not AWS_ON_DEMAND_PRICES[region][product]:
print "WARNING: Empty %s:%s" % (region, product)
continue
if product not in vals[reg_key][zone]:
vals[reg_key][zone][product] = {}
tss[reg_key][zone][product] = {}
for inst_type in common.AWS_ON_DEMAND_PRICES[region][product]:
# print "%s/%s/%s/%s" % (reg_key, zone, product, inst_type)
vals[reg_key][zone][product][inst_type] = []
tss[reg_key][zone][product][inst_type] = []
#sys.exit(1)
for region in ec2.region_names:
all_regions.add(region)
cnt = 0
next_token = None
print 'Collecting spot prices from region: %s for %s to %s' % (region, start_time.format(_FMT), end_time.format(_FMT))
sys.stdout.flush()
# if region != 'us-east-1':
#continue
while True:
endpoint = ec2.get_endpoint(region)
if next_token:
response, data = operation.call(
endpoint,
start_time=start_time.format(_FMT),
end_time=end_time.format(_FMT),
next_token=next_token,
)
else:
response, data = operation.call(
endpoint,
start_time=start_time.format(_FMT),
)
next_token = data.get('NextToken')
spot_data = data.get('SpotPriceHistory', [])
first_entry_in_batch = True
sys.stdout.flush()
for d in spot_data:
ts = common.ts_from_aws(d)
if first_entry_in_batch:
print "Fetched %s records starting with %s" % (len(spot_data), d['Timestamp'])
first_entry_in_batch = False
# {u'Timestamp': '2014-04-10T23:49:21.000Z', u'ProductDescription': 'Linux/UNIX (Amazon VPC)', u'InstanceType': 'hi1.4xlarge', u'SpotPrice': '0.128300', u'AvailabilityZone': 'us-east-1b'}
reg_key = region.replace('-','_')
d['Region'] = reg_key
d['InstanceTypeNorm'] = d['InstanceType'].replace('.','_')
value = d['SpotPrice']
zone = d['AvailabilityZone'].replace('-','_')
product = d['ProductDescription'].replace('-','_').replace('(','').replace(')','_').replace(' ','_').replace('/','_')
if product.endswith('_'):
product=product[:-1]
inst_type = d['InstanceTypeNorm'].replace('-','_')
tags = {
'cloud' : 'aws',
'region' : reg_key,
'zone' : zone,
'product' : product,
'inst_type' : inst_type,
'units' : 'USD'
}
try:
vals[reg_key][zone][product][inst_type].append(value)
tss[reg_key][zone][product][inst_type].append(ts)
except KeyError:
print "No on-demand info for %s/%s/%s/%s" % (reg_key,zone,product,inst_type)
common.otsdb_send('price_spot', value, tags, ts, False)
tags['price_type'] = 'spot'
common.otsdb_send('price', value, tags, ts, False)
cnt += 1
if not next_token:
break
print "Found %s price points" % cnt
for zone in tss[reg_key]:
for product in tss[reg_key][zone]:
for inst_type in tss[reg_key][zone][product]:
if not tss[reg_key][zone][product][inst_type]:
print "No spot info for %s/%s/%s/%s." % (reg_key, zone, product, inst_type)
continue
print "%s/%s/%s/%s" % (reg_key, zone, product, inst_type)
tags = {
'cloud' : 'aws',
'region' : reg_key,
'zone' : zone,
'product' : product,
'inst_type' : inst_type,
'units' : 'USD'
}
tss_ts = tss[reg_key][zone][product][inst_type]
tss_ts.sort()
tss_dt = to_datetime(tss_ts, unit='s')
s_data = vals[reg_key][zone][product][inst_type]
s1 = Series(s_data, tss_dt)
# print "Creating Series(%s, %s) from %s; length: %s" % (s_data, tss_dt, tss_ts, len(s1))
if len(s1) > 1:
# We already took care of 1-length (no fill)
s2 = s1.asfreq('1Min', method='ffill')
# print "Sparse series:\n%s\n" % s1
# print "Filled series:\n%s\n" % s2
# print "Sparse: %s, filled: %s" % (len(s1), len(s2))
for (dt,value) in s2.iteritems():
ts = arrow.Arrow.fromdatetime(dt).timestamp
common.otsdb_send('price_spot', value, tags, ts, False)
tags['price_type'] = 'spot'
common.otsdb_send('price', value, tags, ts, False)
sys.stdout.flush()
