def main():
(opts, source_region, source_image_id, image_name,
image_arch) = parse_args()
# Validate AMI
conn = EC2Connection(region=ec2.get_region(source_region))
image = conn.get_image(source_image_id)
if not image.is_public:
print >> stderr, ("Image %s is not public, no one will be able to " \
"use it!" % source_image_id)
sys.exit(1)
if opts.files:
if not os.path.exists(opts.base_directory):
os.mkdir(opts.base_directory)
for dest_region in DEST_REGIONS:
try:
region = ec2.get_region(dest_region)
conn = EC2Connection(region=region, validate_certs=False)
except Exception as e:
print >> stderr, (e)
sys.exit(1)
new_image = conn.copy_image(source_region, source_image_id, image_name)
print "Created new image: %s in %s" % (new_image.image_id, dest_region)
if opts.files:
dest_dir = os.path.join(opts.base_directory, dest_region)
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
f = open(os.path.join(dest_dir, image_arch), 'w')
f.write(new_image.image_id)
f.close()
def main():
(opts, source_region, source_image_id, image_name, image_arch) = parse_args()
# Validate AMI
conn = EC2Connection(region=ec2.get_region(source_region))
image = conn.get_image(source_image_id)
if not image.is_public:
print >> stderr, ("Image %s is not public, no one will be able to " \
"use it!" % source_image_id)
sys.exit(1)
if opts.files:
if not os.path.exists(opts.base_directory):
os.mkdir(opts.base_directory)
for dest_region in DEST_REGIONS:
try:
region = ec2.get_region(dest_region)
conn = EC2Connection(region=region, validate_certs=False)
except Exception as e:
print >> stderr, (e)
sys.exit(1)
new_image = conn.copy_image(source_region, source_image_id, image_name)
print "Created new image: %s in %s" % (new_image.image_id, dest_region)
if opts.files:
dest_dir = os.path.join(opts.base_directory, dest_region)
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
f = open(os.path.join(dest_dir, image_arch), 'w')
f.write(new_image.image_id)
f.close()
def get_running_instances(access_key=None, secret_key=None, security_group=None, region=None):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
if region is None:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
else:
ec2_region_list = [get_region(region)]
for region in ec2_region_list:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region)
running_instances = []
try:
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances.extend([i for i in s.instances() if i.state == 'running'])
except boto.exception.EC2ResponseError:
logging.error('Region [' + region.name + '] inaccessible')
if running_instances:
for instance in running_instances:
instances_all_regions_list.append(instance)
return instances_all_regions_list
def _upload_credentials(aws_config, manager_config_path):
temp_config = tempfile.mktemp()
credentials = ConfigParser()
credentials.add_section('Credentials')
credentials.set('Credentials', 'aws_access_key_id',
aws_config['aws_access_key_id'])
credentials.set('Credentials', 'aws_secret_access_key',
aws_config['aws_secret_access_key'])
if aws_config.get('ec2_region_name'):
region = get_region(aws_config['ec2_region_name'])
credentials.add_section('Boto')
credentials.set('Boto', 'ec2_region_name',
aws_config['ec2_region_name'])
credentials.set('Boto', 'ec2_region_endpoint',
region.endpoint)
credentials_string = StringIO()
credentials.write(credentials_string)
with open(temp_config, 'w') as temp_config_file:
temp_config_file.write(credentials_string.getvalue())
make_default_lower = \
'sed -i "s/\[DEFAULT\]/\[default\]/g" {0}'.format(
constants.AWS_DEFAULT_CONFIG_PATH)
fabric.api.put(temp_config, manager_config_path)
fabric.api.run(make_default_lower)
def save(self, *args, **kwargs):
"""
Register a new security group is not existing at AWS
"""
# Set the region
self.region = self.availability_zone[:-1]
# Check if a security group exists
cloud = Cloud.objects.get(id=self.cloud.id)
self.security_group = "Jeeves_%s_%s" % (cloud.uuid, self.role.id)
connection = ec2.connection.EC2Connection(
aws_access_key_id=cloud.aws_id,
aws_secret_access_key=cloud.aws_secret,
region=ec2.get_region(self.region))
is_registered = False
for registered_sg in connection.get_all_security_groups():
if self.security_group == registered_sg.name:
is_registered = True
# Register new Security group
if not is_registered:
print "Security group %s not found. Creating it." % self.security_group
security_group = connection.create_security_group(
self.security_group, self.role.name)
security_group.authorize(ip_protocol='icmp', cidr_ip='0.0.0.0/0')
security_group.authorize('tcp', 22, 22, '0.0.0.0/0')
# Save the object
super(Instance, self).save(*args, **kwargs)
def get_running_instances(access_key=None,
secret_key=None,
security_group=None,
region=None):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
if region is None:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
else:
ec2_region_list = [get_region(region)]
for region in ec2_region_list:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region)
running_instances = []
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances.extend(
[i for i in s.instances() if i.state == 'running'])
if running_instances:
for instance in running_instances:
instances_all_regions_list.append(instance)
return instances_all_regions_list
def client(self, aws_config=None):
"""Represents the VPCConnection Client
"""
aws_config_property = (self._get_aws_config_property(aws_config)
or self._get_aws_config_from_file())
if not aws_config_property:
return VPCConnection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
if 'ec2_region_name' in aws_config:
del (aws_config['ec2_region_name'])
# for backward compatibility,
# delete this key before passing config to Boto
if 'ec2_region_endpoint' in aws_config:
del (aws_config["ec2_region_endpoint"])
return VPCConnection(**aws_config)
def client(self, aws_config=None):
"""Represents the VPCConnection Client
"""
aws_config_property = (self._get_aws_config_property(aws_config) or
self._get_aws_config_from_file())
if not aws_config_property:
return VPCConnection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
if 'ec2_region_name' in aws_config:
del(aws_config['ec2_region_name'])
# for backward compatibility,
# delete this key before passing config to Boto
if 'ec2_region_endpoint' in aws_config:
del(aws_config["ec2_region_endpoint"])
return VPCConnection(**aws_config)
def start_instance(self):
"""
Start the Instance in the AWS cloud
"""
# Connect to EC2
connection = ec2.connection.EC2Connection(
aws_access_key_id=self.cloud.aws_id,
aws_secret_access_key=self.cloud.aws_secret,
region=ec2.get_region(self.region))
# Reserve Instance
reservation = connection.run_instances(
definitions.AMI['ubuntu-12.04-64bit'][self.region],
instance_type=self.instance_type,
placement=self.availability_zone,
security_groups=[self.security_group],
key_name=self.cloud.aws_key_pair)
# Only one Instance is returned by default
instance = reservation.instances[0]
# Save the Instance information locally
self.instance_id = instance.id
self.aws_status = instance.state
self.save()
return instance
def _upload_credentials(aws_config, manager_config_path):
temp_config = tempfile.mktemp()
credentials = ConfigParser()
credentials.add_section('Credentials')
credentials.set('Credentials', 'aws_access_key_id',
aws_config['aws_access_key_id'])
credentials.set('Credentials', 'aws_secret_access_key',
aws_config['aws_secret_access_key'])
if aws_config.get('ec2_region_name'):
region = get_region(aws_config['ec2_region_name'])
credentials.add_section('Boto')
credentials.set('Boto', 'ec2_region_name',
aws_config['ec2_region_name'])
credentials.set('Boto', 'ec2_region_endpoint', region.endpoint)
credentials_string = StringIO()
credentials.write(credentials_string)
with open(temp_config, 'w') as temp_config_file:
temp_config_file.write(credentials_string.getvalue())
make_default_lower = \
'sed -i "s/\[DEFAULT\]/\[default\]/g" {0}'.format(
constants.AWS_DEFAULT_CONFIG_PATH)
fabric.api.put(temp_config, manager_config_path)
fabric.api.run(make_default_lower)
def main():
farmDict={}
connectionstring=""
numArgs=len(sys.argv)
if numArgs == 1:
sys.exit()
VPCName=sys.argv[1]
VPCName = VPCName.lower()
#farmList = open("/home/ansible/staging/files/farms.txt", "w")
connStrFL = open("/home/ansible/staging/files/connStrSVT.txt", "w")
if VPCName == "staging":
cidrBlock = "172.0.0.0/16"
###Find the VPC
myregion = ec2.get_region(region_name='eu-central-1')
c = vpc.VPCConnection(region=myregion)
vpcs = c.get_all_vpcs(filters=[("cidrBlock", cidrBlock)])
##create an ec2 connection
conn = ec2.connect_to_region('eu-central-1')
instList = conn.get_only_instances(filters={'vpc_id':vpcs[0].id})
for i in instList:
if 'farmName' in i.tags:
frmName = i.tags['farmName']
farmDict[frmName] = frmName
connectionstring = connectionstring + "," + i.private_ip_address + ":27017"
for fList in farmDict:
print fList
connStrFL.write(connectionstring)
connStrFL.close()
def terminate_instance(self):
"""
Stop a running AWS Instance
"""
# Connect to EC2
connection = ec2.connection.EC2Connection(
aws_access_key_id=self.cloud.aws_id,
aws_secret_access_key=self.cloud.aws_secret,
region=ec2.get_region(self.region))
# Get the Instance object
reservations = connection.get_all_instances(
instance_ids=[self.instance_id])
# Messed up syntax, but we are taking the only instance object from a
# multidimentional array
instance = reservations[0].instances[0]
# Terminate the instance
instance.terminate()
# Set the local state
instance.update()
self.aws_status = instance.state
self.save()
return True
def Bill(account,id):
billingkey = os.environ.get('%sBill_Key' % account)
billingsecret = os.environ.get('%sBill_Secret' % account)
myRegion = get_region('us-east-1') #create cloudwatch for billing alert first. the default area is us-east-1
conn = boto.ec2.cloudwatch.connect_to_region('us-east-1',aws_access_key_id=billingkey,aws_secret_access_key=billingsecret)
Metrics=conn.list_metrics(metric_name=u'EstimatedCharges',namespace=u'AWS/Billing')
#Last = datetime.datetime(2015,3,26)
#Start = datetime.datetime(2015,3,20)
Last = datetime.date.today()
#Last = datetime.date.today()+timedelta(days=-5)
during = timedelta(days=-1)
Start = Last + during
file="%s_total.out" % account
f=open(file,'w')
for metric in Metrics:
if [id] in metric.dimensions.values() and u'ServiceName' not in metric.dimensions :
#print Start,metric.query(Start,Last,'Average',period=86400)
f.write(str(Start)+'\t'+str(metric.query(Start,Last,'Average',period=86400))+'\n')
f.close()
cost_cmd="echo `cat %s |awk -F':' '{print $3}'|awk -F'.' '{print $1}'`" % file
date_cmd="echo `awk '{print $1}' %s`" % file
cost=int(os.popen(cost_cmd).read().strip('\n'))
date=str(os.popen(date_cmd).read().strip('\n'))
conn = MySQLdb.connect(user='******', db='instance', passwd='', host='localhost')
cursor = conn.cursor()
sql="insert into billing_info_%s(date,total_all) values('%s',%s)" % (account,date,cost)
cursor.execute(sql)
conn.commit()
cursor.close()
conn.close()
def __init__(self, **kwargs):
username = kwargs.get('username')
password = kwargs.get('password')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.kwargs = kwargs
def client(self):
"""Represents the ELBConnection Client
"""
aws_config_property = (self._get_aws_config_property()
or self._get_aws_config_from_file())
if not aws_config_property:
return ELBConnection()
aws_config = aws_config_property.copy()
if aws_config_property.get('elb_region_name') and \
aws_config_property.get('elb_region_endpoint'):
region_object = \
get_region(aws_config_property['elb_region_name'])
region_object.endpoint = \
aws_config_property['elb_region_endpoint']
aws_config['region'] = region_object
elif aws_config_property.get('elb_region_name') and \
not aws_config_property.get('elb_region_endpoint'):
aws_config['region'] = aws_config_property['elb_region_name']
aws_config = self.aws_config_cleanup(aws_config)
if 'region' in aws_config:
if type(aws_config['region']) is RegionInfo:
return ELBConnection(**aws_config)
elif type(aws_config['region']) is str:
elb_region = aws_config.pop('region')
return connect_to_elb_region(elb_region, **aws_config)
raise NonRecoverableError(
'Cannot connect to ELB endpoint. '
'You must either provide elb_region_name or both '
'elb_region_name and elb_region_endpoint.')
def __init__(self, **kwargs):
super(EC2System, self).__init__(kwargs)
username = kwargs.get('username')
password = kwargs.get('password')
connection_config = Config(
signature_version='s3v4',
retries=dict(
max_attempts=10
)
)
regionname = kwargs.get('region')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.sqs_connection = connection.SQSConnection(username, password, region=_regions(
regionmodule=sqs, regionname=regionname))
self.elb_connection = ELBConnection(username, password, region=_regions(
regionmodule=elb, regionname=regionname))
self.s3_connection = boto3.resource('s3', aws_access_key_id=username,
aws_secret_access_key=password, region_name=regionname, config=connection_config)
self.ec2_connection = boto3.client('ec2', aws_access_key_id=username,
aws_secret_access_key=password, region_name=regionname, config=connection_config)
self.stackapi = CloudFormationConnection(username, password, region=_regions(
regionmodule=cloudformation, regionname=regionname))
self.cloudformation_connection = boto3.client('cloudformation', aws_access_key_id=username,
aws_secret_access_key=password, region_name=regionname, config=connection_config)
self.sns_connection = boto3.client('sns', region_name=regionname)
self.kwargs = kwargs
def create_provider_image(self):
try:
ec2region = ec2.get_region(self.target['host'], aws_access_key_id=self.provider_user, aws_secret_access_key=self.provider_key)
self.boto = ec2region.connect(aws_access_key_id=self.provider_user, aws_secret_access_key=self.provider_key)
instance = Threaded_create_instance(ami_id=self.target[self.tdl.xpathEval("/template/os/arch")[0].content], image_id=self.target['host'], hwp_id='t1.micro', boto_connection=self.boto)
if self.proxy_ami_id:
proxy = Threaded_create_instance(self.proxy_ami_id, image_id=self.target['host'], hwp_id='t1.micro', boto_connection=self.boto )
proxy.start()
instance.start()
proxy.join()
instance.join()
self.proxy_id, self.proxy_address, self.proxy_password = proxy.instance.id, proxy.instance_address, proxy.instance_password
self.instance_id, self.instance_address, self.instance_password = instance.instance.id, instance.instance_address, instance.instance_password
else:
self.proxy_address, self.proxy_password = self.proxy_address, self.proxy_password
instance.start()
instance.join()
self.instance_id, self.instance_address, self.instance_password = instance.instance.id, instance.instance_address, instance.instance_password
return self.customize()
except:
raise ImageFactoryException("Invalid create image requested for target %s" % (self.target))
def main():
parser = OptionParser()
parser.add_option("-c", "--config", help="Boto configuration (default: /etc/mboto.cfg)", dest="keycfg", default="/etc/mboto.cfg")
parser.add_option("-a", "--account", help="The aws account from boto config", dest="account")
parser.add_option("-r", "--region", help="Region: ue, uw2, as, etc. (default: ue)", dest="region", default="ue")
parser.add_option("-H", "--headers", help="Set headers - use 'T:tagname' for including tags; use 'Status=running' to filter; use 'T:Name.filter=6016 to filter more. ", default="ID,Zone,State,PublicIP,PrivateIP,T:Name,T:Role", action="store", dest="headers", metavar="ID,Zone,State,PublicIP,PrivateIP,T:Name,T:Role")
(options, args) = parser.parse_args()
result, awskeys = zkParser(options.keycfg)
if not result:
print "%s is corrupt..." % options.keycfg
sys.exit(0)
if options.account in awskeys:
awskey = awskeys[options.account]
else:
print "accounts are not found in %s..." % (options.keycfg)
sys.exit(0)
# Connect the region
if options.region not in REGIONS:
print "Region %s not found." % options.region
sys.exit(1)
ec2conn = boto.connect_ec2(region=get_region(REGIONS[options.region]),
aws_access_key_id=awskey['aws_access_key_id'],
aws_secret_access_key=awskey['aws_secret_access_key'])
# Read headers
if options.headers:
headers = tuple(options.headers.split(','))
else:
headers = ("ID", 'Zone', "Groups", "Hostname")
# Create format string
format_string = ""
for h in headers:
if h.startswith('T:'):
format_string += "%%-%ds" % HEADERS['T:']['length']
elif '.filter=' in h:
format_string += "%%-%ds" % HEADERS[h.split('.filter=')[0]]['length']
elif '=' in h:
format_string += "%%-%ds" % HEADERS[h.split('=')[0]]['length']
else:
format_string += "%%-%ds" % HEADERS[h]['length']
# List and print
print format_string % headers
print "-" * len(format_string % headers)
for r in ec2conn.get_all_instances():
groups = [g.name for g in r.groups]
for i in r.instances:
i.groups = ','.join(groups)
prints = []
for h in headers:
key, value = get_column(h, i)
if value:
prints.append(value)
if len(prints) == len(headers):
print format_string % tuple(prints)
def _upload_credentials(aws_config, manager_config_path):
__, temp_config = tempfile.mkstemp()
credentials = ConfigParser()
credentials.add_section('Credentials')
credentials.set('Credentials', 'aws_access_key_id',
aws_config['aws_access_key_id'])
credentials.set('Credentials', 'aws_secret_access_key',
aws_config['aws_secret_access_key'])
if aws_config.get('ec2_region_name'):
region = get_region(aws_config['ec2_region_name'])
credentials.add_section('Boto')
credentials.set('Boto', 'ec2_region_name',
aws_config['ec2_region_name'])
credentials.set('Boto', 'ec2_region_endpoint',
region.endpoint)
with open(temp_config, 'w') as temp_config_file:
credentials.write(temp_config_file)
fabric.api.sudo('mkdir -p {0}'.
format(os.path.dirname(manager_config_path)))
fabric.api.put(temp_config,
manager_config_path,
use_sudo=True)
def _get_aws_config(self):
region = get_region(self.env.ec2_region_name)
return {
'aws_access_key_id': self.env.aws_access_key_id,
'aws_secret_access_key': self.env.aws_secret_access_key,
'region': region
}
def __init__(self, **kwargs):
username = kwargs.get('username')
password = kwargs.get('password')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.stackapi = CloudFormationConnection(username, password)
self.kwargs = kwargs
def _get_aws_config(self):
region = get_region(self.env.ec2_region_name)
return {
'aws_access_key_id': self.env.aws_access_key_id,
'aws_secret_access_key': self.env.aws_secret_access_key,
'region': region
}
def __init__(self, **kwargs):
super(EC2System, self).__init__(kwargs)
username = kwargs.get('username')
password = kwargs.get('password')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.stackapi = CloudFormationConnection(username, password)
self.kwargs = kwargs
def stop_group(self, access, secret, group):
conn = ec2.get_region('sa-east-1').connect()
for reservation in conn.get_all_instances():
for instance in reservation.instances:
tags = instance.tags
if 'Group' in tags.keys():
if group in tags['Group']:
if 'terminated' not in instance.state:
instance.terminate()
def _client_credentials(self):
region = get_region(self.env.ec2_region_name)
return {
'aws_access_key_id': self.env.aws_access_key_id,
'aws_secret_access_key': self.env.aws_secret_access_key,
'region': region
}
def __init__(self,region_name, key, secret):
"""establish a connection with ec2"""
self.region_name = region_name
self.key = key
self.secret = secret
self.region = ec2.get_region(self.region_name, aws_access_key_id=self.key,
aws_secret_access_key=self.secret)
self.connection = self.region.connect(aws_access_key_id=self.key,
aws_secret_access_key=self.secret)
def _client_credentials(self):
region = get_region(self.env.ec2_region_name)
return {
'aws_access_key_id': self.env.aws_access_key_id,
'aws_secret_access_key': self.env.aws_secret_access_key,
'region': region
}
def reload_settings(self, settings=None):
"""
(re)-configure instance creator. Useful if you changed your settings
"""
self.s = settings or Settings(self.settings_filename)
self.region = get_region(self.s.EC2_REGION)
self.conn = EC2Connection(self.s.AWS_ACCESS_KEY_ID,
self.s.AWS_SECRET_ACCESS_KEY,
region=self.region)
self.settings_filename = self.s.filename
def reload_settings(self, settings=None):
"""
(re)-configure instance creator. Useful if you changed your settings
"""
self.s = settings or Settings(self.settings_filename)
self.region = get_region(self.s.EC2_REGION)
self.conn = EC2Connection(self.s.AWS_ACCESS_KEY_ID,
self.s.AWS_SECRET_ACCESS_KEY,
region=self.region)
self.settings_filename = self.s.filename
def getConnection(): r = ec2.get_region(region_name=AWS_CONFIG['AWS_REGION'], aws_access_key_id=AWS_CONFIG['AWS_KEY'], aws_secret_access_key=AWS_CONFIG['AWS_SECRET']) print "Connecting to region %s" % r conn = ec2.connection.EC2Connection(region=r, aws_access_key_id=AWS_CONFIG['AWS_KEY'], aws_secret_access_key=AWS_CONFIG['AWS_SECRET']) return conn
def _get_aws_config(self, set_boto_region=False):
aws_config = {
'aws_access_key_id': self.env.aws_access_key_id,
'aws_secret_access_key': self.env.aws_secret_access_key
}
if set_boto_region:
aws_config['region'] = get_region(self.env.ec2_region_name)
else:
aws_config['ec2_region_name'] = self.env.ec2_region_name
return aws_config
def createConnection():
# Check that the required settings are present
checkRequiredSettings([
'aws_region',
'aws_access_key',
'aws_access_secret',
])
# Connect to the region
region = ec2.get_region(settings['aws_region'])
return EC2Connection(settings['aws_access_key'], settings['aws_access_secret'], region=region)
def _get_aws_config(self, set_boto_region=False):
aws_config = {
'aws_access_key_id': self.env.aws_access_key_id,
'aws_secret_access_key': self.env.aws_secret_access_key
}
if set_boto_region:
aws_config['region'] = get_region(self.env.ec2_region_name)
else:
aws_config['ec2_region_name'] = self.env.ec2_region_name
return aws_config
def _get_ec2_conn(self, region):
"""
Get boto's EC2 connection
:type region: string
:rtype: boto.ec2.EC2Connection
"""
return ec2.EC2Connection(
region=ec2.get_region(region),
aws_access_key_id=self.aws_key,
aws_secret_access_key=self.aws_secret
)
def start_group(self, access, secret, group, how_many):
conn = ec2.get_region('sa-east-1').connect()
image = conn.get_image('ami-ee35eaf3')
reservation = image.run(min_count=how_many, max_count=how_many,
user_data=group, instance_type='m1.large', key_name='cloudia',
security_groups=['cloudia'])
i = 1
for instance in reservation.instances:
instance.add_tag('Group', value=group)
instance.add_tag('Name', value='%s - %d' % (group, i))
i += 1
def __init__(self, **kwargs):
super(EC2System, self).__init__(**kwargs)
self._username = kwargs.get('username')
self._password = kwargs.get('password')
connection_config = Config(signature_version='s3v4',
retries=dict(max_attempts=10))
self._region_name = kwargs.get('region')
self._region = get_region(self._region_name)
self.api = EC2Connection(self._username,
self._password,
region=self._region)
self.sqs_connection = boto3client('sqs',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.elb_connection = ELBConnection(self._username,
self._password,
region=_regions(
regionmodule=elb,
regionname=self._region_name))
self.s3_connection = boto3resource(
's3',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.ec2_connection = boto3client('ec2',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.ecr_connection = boto3client('ecr',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.cloudformation_connection = boto3client(
'cloudformation',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.sns_connection = boto3client('sns', region_name=self._region_name)
self.kwargs = kwargs
def get_running_instances(access_key=None,
secret_key=None,
security_group=None,
region=None,
safe_mode=False,
delay=0):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
if region is None:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
else:
ec2_region_list = [get_region(region)]
for region in ec2_region_list:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region)
running_instances = []
date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
try:
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances.extend([
i for i in s.instances() if i.state == 'running' and
(datetime.utcnow() - datetime.strptime(
i.launch_time, date_format)).seconds > delay
])
except boto.exception.EC2ResponseError:
logging.error('Region [' + region.name + '] inaccessible')
if safe_mode:
logging.error(
'Safe mode enabled. No new haproxy cfg is generated. Exit now.'
)
exit(1)
if running_instances:
for instance in running_instances:
logging.info(instance)
print(instance)
print("============================")
instances_all_regions_list.append(instance)
logging.info("**********************************")
logging.info(instances_all_regions_list)
instances_all_regions_list_sorted = sorted(instances_all_regions_list)
print(instances_all_regions_list_sorted)
return instances_all_regions_list_sorted
def RI(area):
s3key = os.environ.get('S3KEY')
s3secret = os.environ.get('S3SECRET')
myRegion = get_region(area)
#print "myRegion is %s" % myRegion
conn = EC2Connection(aws_access_key_id=s3key,aws_secret_access_key=s3secret,region=myRegion)
result=conn.get_all_reserved_instances()
f=open('RI_out.log','a')
for i in result:
#print i.__dict__
f.write("\t".join([str(i.instance_type),str(i.instance_count),i.availability_zone,i.start,str(i.duration),i.state])+'\n')
f.close()
def _connect(self, provider):
cfg_region = provider.get('region', 'us-east-1')
with config_key_error():
region = ec2.get_region(cfg_region,
aws_access_key_id=provider['id'],
aws_secret_access_key=provider['key'])
self.conn = boto.connect_ec2(
aws_access_key_id=provider['id'],
aws_secret_access_key=provider['key'],
region=region)
def ec2_digger():
zones = ['us-east-1', 'us-west-2', 'us-west-1']
server_dict = dict()
tag_dict = dict()
for zone in zones:
ec2_conn = EC2Connection(region=get_region(zone))
reservation = ec2_conn.get_all_instances()
if len(reservation) <= 0:
print red("No instances in " + zone + ", cool cat.")
else:
for r in reservation:
for i in r.instances:
instance = dict()
status = i.update()
if status == "running":
instance['status'] = status
instance['zone'] = zone
if hasattr(i, 'tags'):
tags = getattr(i, 'tags')
for tag in tags:
if tag.lower() not in tag_dict:
tag_dict[tag.lower()] = dict()
tag_vals = tag_dict[tag.lower()]
tag_val = tags.get(tag)
if tag_val.lower() not in tag_vals:
tag_vals[tag_val.lower()] = []
tag_vals[tag_val.lower()].append(getattr(i, 'dns_name'))
methods = dir(i)
for method in methods:
if not method.startswith("_"):
if not hasattr(getattr(i, method), "__call__"):
if method == 'block_device_mapping':
blocks = getattr(i, method)
block_dict = dict()
for device in blocks:
block_dict[device] = blocks.get(device).volume_id
instance[method] = block_dict
elif method == 'groups':
groups = getattr(i, method)
new_groups = list()
for group in groups:
new_groups.append(group.name)
instance[method] = new_groups
else:
instance[method] = getattr(i, method)
server_dict[instance.get('dns_name')] = instance
return server_dict, tag_dict
def list_groups(self, access, secret):
conn = ec2.get_region('sa-east-1').connect()
groups = {}
for reservation in conn.get_all_instances():
for instance in reservation.instances:
if 'terminated' not in instance.state:
tags = instance.tags
if 'Group' in tags.keys():
group_name = tags['Group']
if group_name not in groups.keys():
groups[group_name] = Group(group_name)
groups[group_name].add_instance(instance)
return groups
def __init__(self, **kwargs):
super(EC2System, self).__init__(kwargs)
username = kwargs.get('username')
password = kwargs.get('password')
regionname = kwargs.get('region')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.sqs_connection = connection.SQSConnection(username, password, region=_regions(
regionmodule=sqs, regionname=regionname))
self.elb_connection = ELBConnection(username, password, region=_regions(
regionmodule=elb, regionname=regionname))
self.s3_connection = boto.connect_s3(username, password)
self.stackapi = CloudFormationConnection(username, password, region=_regions(
regionmodule=cloudformation, regionname=regionname))
self.kwargs = kwargs
def delete(self, *args, **kwargs):
"""
Delete associated services
"""
# If there are no more Instances in the Role, remove the security group
if len(Instance.objects.filter(role=self.role.id,
region=self.region)) <= 1:
cloud = Cloud.objects.get(id=self.cloud.id)
connection = ec2.connection.EC2Connection(
aws_access_key_id=cloud.aws_id,
aws_secret_access_key=cloud.aws_secret,
region=ec2.get_region(self.region))
connection.delete_security_group(self.security_group)
print "Removed security group %s" % self.security_group
# Save the object
super(Instance, self).delete(*args, **kwargs)
def Bill(account, id):
billingkey = os.environ.get('%sBill_Key' % account)
billingsecret = os.environ.get('%sBill_Secret' % account)
myRegion = get_region(
'us-east-1'
) #create cloudwatch for billing alert first. the default area is us-east-1
conn = boto.ec2.cloudwatch.connect_to_region(
'us-east-1',
aws_access_key_id=billingkey,
aws_secret_access_key=billingsecret)
Metrics = conn.list_metrics(metric_name=u'EstimatedCharges',
namespace=u'AWS/Billing')
#Last = datetime.datetime(2015,3,26)
#Start = datetime.datetime(2015,3,20)
Last = datetime.date.today()
#Last = datetime.date.today()+timedelta(days=-5)
during = timedelta(days=-1)
Start = Last + during
file = "%s_total.out" % account
f = open(file, 'w')
for metric in Metrics:
if [id] in metric.dimensions.values(
) and u'ServiceName' not in metric.dimensions:
#print Start,metric.query(Start,Last,'Average',period=86400)
f.write(
str(Start) + '\t' +
str(metric.query(Start, Last, 'Average', period=86400)) + '\n')
f.close()
cost_cmd = "echo `cat %s |awk -F':' '{print $3}'|awk -F'.' '{print $1}'`" % file
date_cmd = "echo `awk '{print $1}' %s`" % file
cost = int(os.popen(cost_cmd).read().strip('\n'))
date = str(os.popen(date_cmd).read().strip('\n'))
conn = MySQLdb.connect(user='******',
db='instance',
passwd='',
host='localhost')
cursor = conn.cursor()
sql = "insert into billing_info_%s(date,total_all) values('%s',%s)" % (
account, date, cost)
cursor.execute(sql)
conn.commit()
cursor.close()
conn.close()
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = self._get_aws_config_property()
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get("ec2_region_name"):
region_object = get_region(aws_config_property["ec2_region_name"])
aws_config = aws_config_property.copy()
aws_config["region"] = region_object
del (aws_config["ec2_region_name"])
else:
aws_config = aws_config_property.copy()
return EC2Connection(**aws_config)
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = self._get_aws_config_property()
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
aws_config['region'] = region_object
del (aws_config['ec2_region_name'])
else:
aws_config = aws_config_property.copy()
return EC2Connection(**aws_config)
def get_recent_ami(aws_profile,
filters={},
owner='self',
region='us-east-1',
executable_by='self'):
conn = boto.connect_ec2(profile_name=aws_profile,
region=get_region(region))
# Filter images by owned by self first.
images = conn.get_all_images(owners=owner, filters=filters)
# If no images are owned by self, look for images self can execute.
if not images:
images = conn.get_all_images(executable_by=executable_by,
filters=filters)
# Make sure RC images are omitted from results
images = filter(lambda i: True if '.rc-' not in i.name else False, images)
return sorted(images, key=lambda i: i.creationDate, reverse=True)[0].id
def get_running_instances(access_key=None, secret_key=None, security_group=None, region=None, safe_mode=False, delay=0):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
if region is None:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
else:
ec2_region_list = [get_region(region)]
for region in ec2_region_list:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region)
running_instances = []
date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
try:
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances.extend([
i for i in s.instances()
if i.state == 'running' and (
datetime.utcnow() - datetime.strptime(i.launch_time, date_format)).seconds > delay
])
except boto.exception.EC2ResponseError:
logging.error('Region [' + region.name + '] inaccessible')
if safe_mode:
logging.error('Safe mode enabled. No new haproxy cfg is generated. Exit now.')
exit(1)
if running_instances:
for instance in running_instances:
instances_all_regions_list.append(instance)
return instances_all_regions_list
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = (self._get_aws_config_property() or
self._get_aws_config_from_file())
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
aws_config = self.aws_config_cleanup(aws_config)
return EC2Connection(**aws_config)
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = (self._get_aws_config_property()
or self._get_aws_config_from_file())
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
aws_config = self.aws_config_cleanup(aws_config)
return EC2Connection(**aws_config)
def client(self):
"""Represents the ELBConnection Client
"""
aws_config_property = (self._get_aws_config_property() or
self._get_aws_config_from_file())
if not aws_config_property:
return ELBConnection()
aws_config = aws_config_property.copy()
if aws_config_property.get('elb_region_name') and \
aws_config_property.get('elb_region_endpoint'):
region_object = \
get_region(aws_config_property['elb_region_name'])
region_object.endpoint = \
aws_config_property['elb_region_endpoint']
aws_config['region'] = region_object
elif aws_config_property.get('elb_region_name') and \
not aws_config_property.get('elb_region_endpoint'):
aws_config['region'] = aws_config_property['elb_region_name']
aws_config = self.aws_config_cleanup(aws_config)
if 'region' in aws_config:
if type(aws_config['region']) is RegionInfo:
return ELBConnection(**aws_config)
elif type(aws_config['region']) is str:
elb_region = aws_config.pop('region')
return connect_to_elb_region(
elb_region, **aws_config)
raise NonRecoverableError(
'Cannot connect to ELB endpoint. '
'You must either provide elb_region_name or both '
'elb_region_name and elb_region_endpoint.')
def ec2_connect(region=None):
"""Helper to connect to Amazon Web Services EC2, using identify provided
by environment, as also optional region in arguments.
"""
if not os_environ.get("AWS_ACCESS_KEY_ID", None):
raise EC2LibraryError("Environment variable AWS_ACCESS_KEY_ID is not set.")
if not os_environ.get("AWS_SECRET_ACCESS_KEY", None):
raise EC2LibraryError("Environment variable AWS_SECRET_ACCESS_KEY is not set.")
if not region:
region = env.get("ec2_region")
region = get_region(region,
aws_access_key_id=os_environ.get("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=os_environ.get("AWS_ACCESS_SECRET_KEY")
)
connection = EC2Connection(
os_environ.get("AWS_ACCESS_KEY_ID"),
os_environ.get("AWS_SECRET_ACCESS_KEY"),
region=region
)
return connection
def ec2_connect(region=None):
"""Helper to connect to Amazon Web Services EC2, using identify provided
by environment, as also optional region in arguments.
"""
if not os_environ.get("AWS_ACCESS_KEY_ID", None):
raise EC2TemplateError("Environment variable AWS_ACCESS_KEY_ID is not set.")
if not os_environ.get("AWS_SECRET_ACCESS_KEY", None):
raise EC2TemplateError("Environment variable AWS_SECRET_ACCESS_KEY is not set.")
if not region:
region = env.get("ec2_region")
region = get_region(region,
aws_access_key_id = os_environ.get("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = os_environ.get("AWS_ACCESS_SECRET_KEY")
)
connection = EC2Connection(
os_environ.get("AWS_ACCESS_KEY_ID"),
os_environ.get("AWS_SECRET_ACCESS_KEY"),
region = region
)
return connection
def _upload_credentials(aws_config, manager_config_path):
__, temp_config = tempfile.mkstemp()
credentials = ConfigParser()
credentials.add_section('Credentials')
credentials.set('Credentials', 'aws_access_key_id',
aws_config['aws_access_key_id'])
credentials.set('Credentials', 'aws_secret_access_key',
aws_config['aws_secret_access_key'])
if aws_config.get('ec2_region_name'):
region = get_region(aws_config['ec2_region_name'])
credentials.add_section('Boto')
credentials.set('Boto', 'ec2_region_name',
aws_config['ec2_region_name'])
credentials.set('Boto', 'ec2_region_endpoint', region.endpoint)
with open(temp_config, 'w') as temp_config_file:
credentials.write(temp_config_file)
fabric.api.sudo('mkdir -p {0}'.format(
os.path.dirname(manager_config_path)))
fabric.api.put(temp_config, manager_config_path, use_sudo=True)
def _get_ec2_client(self):
aws_config = self._get_aws_config()
aws_config.pop('ec2_region_name')
aws_config.pop('elb_region_name')
aws_config['region'] = get_region(self.env.ec2_region_name)
return EC2Connection(**aws_config)
def __init__(self, region, **kwargs):
super(IcsEc2, self).__init__(region=get_region(region), **kwargs)
# Before terminating, the script checks match to "key" of the instances and then if running, terminates the instance.
# Once temrinated the data may be lost!
from boto import ec2
import boto3
import boto_config_orig as boto_config
from pprint import pprint
aws_access_id = boto_config.aws_access_id
aws_access_key = boto_config.aws_access_key
ec21 = boto3.resource('ec2', boto_config.aws_region)
termination_list_instances = []
ec2conn = ec2.connection.EC2Connection(
region=ec2.get_region(boto_config.aws_region))
reservations = ec2conn.get_all_instances()
instances = [i for r in reservations for i in r.instances]
for i in instances:
if str(i.key_name) == "<SPECIFIC KEY ONLY>" and str(
i._state) != "terminated(48)": #CHECK FOR STATUS RUNNING
termination_list_instances.append(str(i.id))
if not termination_list_instances:
print("No instance to be terminated!")
quit()
else:
try:
for i in termination_list_instances:
def __init__(self, region, **kwargs):
self.conn = EC2Connection(region=get_region(region), **kwargs)
self.rds = connect_to_region(region, **kwargs)
def main():
parser = OptionParser()
parser.add_option("-c",
"--config",
help="Boto configuration (default: /etc/mboto.cfg)",
dest="keycfg",
default="/etc/mboto.cfg")
parser.add_option("-a",
"--account",
help="The aws account from boto config",
dest="account")
parser.add_option("-r",
"--region",
help="Region: ue, uw2, as, etc. (default: ue)",
dest="region",
default="ue")
parser.add_option(
"-H",
"--headers",
help=
"Set headers - use 'T:tagname' for including tags; use 'Status=running' to filter; use 'T:Name.filter=6016 to filter more. ",
action="store",
dest="headers",
metavar="ID,Zone,State,PublicIP,PrivateIP,T:Name,T:Role")
(options, args) = parser.parse_args()
result, awskeys = zkParser(options.keycfg)
if not result:
print "%s is corrupt..." % options.keycfg
sys.exit(0)
if options.account in awskeys:
awskey = awskeys[options.account]
else:
print "accounts are not found in %s..." % (options.keycfg)
sys.exit(0)
# Connect the region
if options.region not in REGIONS:
print "Region %s not found." % options.region
sys.exit(1)
ec2conn = boto.connect_ec2(
region=get_region(REGIONS[options.region]),
aws_access_key_id=awskey['aws_access_key_id'],
aws_secret_access_key=awskey['aws_secret_access_key'])
# Read headers
#if not options.role:
# print "Role not found."
# sys.exit(1)
if options.headers:
headers = tuple(options.headers.split(','))
else:
headers = ("T:Role", "ID", "PublicIP")
# Create format string
format_string = ""
for h in headers:
if h.startswith('T:'):
format_string += "%%-%ds" % HEADERS['T:']['length']
elif '.filter=' in h:
format_string += "%%-%ds" % HEADERS[h.split('.filter=')
[0]]['length']
elif '=' in h:
format_string += "%%-%ds" % HEADERS[h.split('=')[0]]['length']
else:
format_string += "%%-%ds" % HEADERS[h]['length']
# List and print
print format_string % headers
print "-" * len(format_string % headers)
#import ConfigParser
#config = ConfigParser.RawConfigParser()
config = "screen -t localhost \n"
#rolef = options.region + ".ini"
rolef = "/root/.screenrc"
open(rolef, 'w').close()
for r in ec2conn.get_all_instances():
groups = [g.name for g in r.groups]
for i in r.instances:
i.groups = ','.join(groups)
prints = []
for h in headers:
key, value = get_column(h, i)
if value:
prints.append(value)
if len(prints) == len(headers):
print format_string % tuple(prints)
config += "screen -t " + \
str(prints[0]) + "-" + str(options.region) + "-" + str(options.account) + \
" ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " + \
"-i " + awskey['key_pair'] + "-" + str(options.region) + ".pem " + \
"ics-user@" + str(prints[2]) + "\n"
with open(rolef, 'ab') as wf:
wf.write(config)
from fabric.api import local, task, env, run, sudo
from fabric.operations import open_shell
from fabric.utils import puts, indent
from fabric.context_managers import prefix, settings, cd
from fabric.contrib.console import confirm
from fabric import colors
import boto
from boto import ec2, rds
from os import path, urandom
from hashlib import sha1
import time
import ConfigParser
import yaml
region = 'us-west-2'
ec2_connection = ec2.get_region(region).connect()
rds_connection = rds.connect_to_region(region)
env.security_group = 'wsgi_test' # wsgi_test
env.instance_type = 't1.micro'
env.keypair = 'wsgi_test'
env.user = '******'
env.key_filename = path.join('keypairs', 'wsgi_test.pem')
@task
def app(name):
env.name = name
env.hosts = get_instance_hostnames(name)
