def make_parallel(action, args, threads, azs, field):
"""Make parallel requests to CROC Cloud."""
process_pool = []
results_queue = Queue()
connection = boto.connect_ec2_endpoint(os.environ["EC2_URL"])
def request_maker(queue, args):
print "Start: {0}".format(datetime.datetime.now())
try:
response = connection.make_request(action, args)
response = response.read()
except Exception as e:
response = e
queue.put((response,))
try:
if azs:
process_pool = [ Process(target=request_maker, args=(results_queue, dict(args, **{field: az})),)
for i in xrange(0, threads) for az in azs ]
else:
process_pool = [ Process(target=request_maker, args=(results_queue, args,))
for i in xrange(0, threads) ]
map(lambda p: p.start(), process_pool)
finally:
map(lambda p: p.join(), process_pool)
return [ results_queue.get()[0] for p in process_pool ]
def connect_ec2(args):
"""
Connect to EC2 API by supplied arguments.
Return EC2 connection object.
"""
# Prepare EC2 connection parameters
if not args.ec2_access_key:
if os.getenv('EC2_ACCESS_KEY'):
args.ec2_access_key = os.getenv('EC2_ACCESS_KEY')
elif os.getenv('AWS_ACCESS_KEY'):
args.ec2_access_key = os.getenv('AWS_ACCESS_KEY')
else:
lg.error("EC2 Access Key not supplied. Use EC2_ACCESS_KEY or AWS_ACCESS_KEY environment variables or command line option")
sys.exit(1)
if not args.ec2_secret_key:
if os.getenv('EC2_SECRET_KEY'):
args.ec2_secret_key = os.getenv('EC2_SECRET_KEY')
elif os.getenv('AWS_SECRET_KEY'):
args.ec2_secret_key = os.getenv('AWS_SECRET_KEY')
else:
lg.error("EC2 Secret Key not supplied. Use EC2_SECRET_KEY or AWS_SECRET_KEY environment variables or command line option")
sys.exit(1)
if not args.ec2_url:
if os.getenv('EC2_URL'):
args.ec2_url = os.getenv('EC2_URL')
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
if args.timeout:
boto.config.set('Boto','http_socket_timeout', str(args.timeout))
if args.cert:
boto.config.set('Boto', 'ca_certificates_file', args.cert)
# Connect to EC2
if args.ec2_url:
# Special connection to EC2-compatible API (eg. OpenStack)
if args.insecure:
validate_certs = False
else:
validate_certs = True
lg.debug("Connecting to EC2: url=%s, validate_certs=%s" % (args.ec2_url, validate_certs))
ec2 = boto.connect_ec2_endpoint(url=args.ec2_url,
validate_certs=validate_certs,
aws_access_key_id=args.ec2_access_key,
aws_secret_access_key=args.ec2_secret_key)
else:
# Standard connection to AWS EC2
ec2 = boto.ec2.connect_to_region(args.ec2_region,
aws_access_key_id=args.ec2_access_key,
aws_secret_access_key=args.ec2_secret_key)
return ec2
def _init_ec2(self):
self.ec2h = boto.connect_ec2_endpoint(
self.cf['ec2']['api_url'],
aws_access_key_id=self.cf['ec2']['aws_access_key_id'],
aws_secret_access_key=self.cf['ec2']['aws_secret_access_key'],
api_version=self.cf['ec2']['api_version'])
self.ec2img = self.ec2_image( self.cf['ec2']['image_id'] )
if self.ec2img is None:
self.logctl.error('Cannot find EC2 image "%s"', self.cf['ec2']['image_id'])
else:
self.logctl.debug('EC2 image "%s" found' % self.cf['ec2']['image_id'])
def __init__(self, configuration=None, access_key=None, secret_key=None, proxy=None, proxy_port=None):
self.proxy = proxy
self.proxy_port = proxy_port
if configuration is not None:
self.load_configuration(configuration)
elif access_key is not None and secret_key is not None:
self.conn = boto.connect_ec2_endpoint(ENDPOINT, access_key, secret_key, proxy=self.proxy,
proxy_port=self.proxy_port)
elif access_key is not None and secret_key is None:
raise ValueError("access_key is set but not secret_key")
elif access_key is None and secret_key is not None:
raise ValueError("secret_key is set but not access_key")
def test_boto_connect(self): conn = boto.connect_ec2_endpoint(config.EC2_URL, aws_access_key_id=config.EC2_ACCESS, aws_secret_access_key=config.EC2_SECRET) nets = conn.get_all_addresses() images = conn.get_all_images() snaps = conn.get_all_snapshots() #conn.run_instances(name) #r = conn.run_instances(BASE_IMAGE) #ins = conn.get_all_instances(instance_ids=[u'i-00000017']) instances = conn.get_all_instances() #conn.terminate_instances(instance_ids=[u'i-00000016']) pass
def load_configuration(self, fp):
"""
Opens a yaml file and reads from it, verifying that it contains at least the two authentication fields we need.
:param string fp: Path to the yaml configuration file
:raises ValueError: When the secret_key and/or access_key are missing in the configuration file.
"""
with open(fp, 'r') as stream:
cnf = yaml.load(stream)
if 'access_key' in cnf and 'secret_key' in cnf:
self.conn = boto.connect_ec2_endpoint(ENDPOINT, cnf['access_key'], cnf['secret_key'], proxy=self.proxy,
proxy_port=self.proxy_port)
else:
raise ValueError("Need access_key and secret_key in {} configuration file".format(fp))
def create_client(self):
"""Return ec2 client."""
import boto
kc = self.keystone()
if kc.version != "v2.0":
raise exceptions.RallyException(
_("Rally EC2 benchmark currently supports only"
"Keystone version 2"))
ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
tenant_id=kc.auth_tenant_id)
client = boto.connect_ec2_endpoint(
url=self._get_endpoint(),
aws_access_key_id=ec2_credential.access,
aws_secret_access_key=ec2_credential.secret,
is_secure=self.credential.insecure)
return client
def main():
ec2_url = os.environ.get('EC2_URL')
ec2_access_key = os.environ.get('EC2_ACCESS_KEY')
ec2_secret_key = os.environ.get('EC2_SECRET_KEY')
ec2_api_version = os.environ.get('EC2_API_VERSION')
print "Environment:"
print "EC2_URL: %s" % ec2_url
print "EC2_ACCESS_KEY: %s" % ec2_access_key
print "EC2_SECRET_KEY: %s" % ec2_secret_key
print "EC2_API_VERSION: %s" % ec2_api_version
if ec2_url is None or ec2_access_key is None or ec2_secret_key is None:
print "You must set all the proper EC2_* envvars for making it work."
exit(1)
# Initialize EC2 connection
ec2h = boto.connect_ec2_endpoint(
ec2_url,
aws_access_key_id=ec2_access_key,
aws_secret_access_key=ec2_secret_key,
api_version=ec2_api_version)
# Try to list VMs and images
try:
res = ec2h.get_all_reservations()
print "\n=== Instances ==="
for r in res:
for i in r.instances:
print "id=%s type=%s name=%s ip=%s key=%s state=%s" % (i.id, i.instance_type, i.public_dns_name, i.private_ip_address, i.key_name, i.state)
img = ec2h.get_all_images()
print "\n=== Images ==="
for im in img:
print "id=%s name=%s" % (im.id, im.name)
except Exception, e:
print "Boto can't talk to EC2: check your credentials"
exit(2)
def create_client(self):
"""Return ec2 client."""
LOG.warning("rally.osclient.EC2 is deprecated since Rally 0.10.0.")
import boto
kc = self.keystone()
if kc.version != "v2.0":
raise exceptions.RallyException(
"Rally EC2 scenario supports only Keystone version 2")
ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
tenant_id=kc.auth_tenant_id)
client = boto.connect_ec2_endpoint(
url=self._get_endpoint(),
aws_access_key_id=ec2_credential.access,
aws_secret_access_key=ec2_credential.secret,
is_secure=self.credential.https_insecure)
return client
def ec2(self):
"""Return ec2 client."""
import boto
kc = self.keystone()
if kc.version != "v2.0":
raise exceptions.RallyException(
_("Rally EC2 benchmark currently supports only"
"Keystone version 2"))
ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
tenant_id=kc.auth_tenant_id)
ec2_api_url = kc.service_catalog.url_for(
service_type=consts.ServiceType.EC2,
endpoint_type=self.endpoint.endpoint_type,
region_name=self.endpoint.region_name)
client = boto.connect_ec2_endpoint(
url=ec2_api_url,
aws_access_key_id=ec2_credential.access,
aws_secret_access_key=ec2_credential.secret,
is_secure=self.endpoint.insecure)
return client
def create_client(self):
"""Return ec2 client."""
import boto
kc = self.keystone()
if kc.version != "v2.0":
raise exceptions.RallyException(
_("Rally EC2 benchmark currently supports only"
"Keystone version 2"))
ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
tenant_id=kc.auth_tenant_id)
ec2_api_url = kc.service_catalog.url_for(
service_type=consts.ServiceType.EC2,
endpoint_type=self.credential.endpoint_type,
region_name=self.credential.region_name)
client = boto.connect_ec2_endpoint(
url=ec2_api_url,
aws_access_key_id=ec2_credential.access,
aws_secret_access_key=ec2_credential.secret,
is_secure=self.credential.insecure)
return client
def ec2_connect(module):
""" Return an ec2 connection"""
region, ec2_url, boto_params = get_aws_connection_info(module)
# If we have a region specified, connect to its endpoint.
if region:
try:
ec2 = connect_to_aws(boto.ec2, region, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
module.fail_json(msg=str(e))
# Otherwise, no region so we fallback to the old connection method
elif ec2_url:
try:
ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="Either region or ec2_url must be specified")
return ec2
def ec2_connect(module):
""" Return an ec2 connection"""
region, ec2_url, boto_params = get_aws_connection_info(module)
# If we have a region specified, connect to its endpoint.
if region:
try:
ec2 = connect_to_aws(boto.ec2, region, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
module.fail_json(msg=str(e))
# Otherwise, no region so we fallback to the old connection method
elif ec2_url:
try:
ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="Either region or ec2_url must be specified")
return ec2
def ec2_connect(module):
""" Return an ec2 connection"""
region, ec2_url, boto_params = get_aws_connection_info(module)
# If ec2_url is present use it
if ec2_url:
try:
ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
module.fail_json(msg=str(e))
# Otherwise, if we have a region specified, connect to its endpoint.
elif region:
try:
ec2 = connect_to_aws(boto.ec2, region, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="Either region or ec2_url must be specified")
return ec2
def isolated_connection_factory(): return boto.connect_ec2_endpoint(config.EC2_URL, aws_secret_access_key=config.EC2_SECRET, aws_access_key_id=config.EC2_ACCESS)
def isolated_connection_factory():
return boto.connect_ec2_endpoint(config.EC2_URL,
aws_secret_access_key=config.EC2_SECRET,
aws_access_key_id=config.EC2_ACCESS)
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(instance=dict(),
id=dict(),
name=dict(),
volume_size=dict(type='int'),
volume_type=dict(choices=['standard', 'gp2', 'io1', 'st1', 'sc1'],
default='standard'),
iops=dict(type='int'),
encrypted=dict(type='bool', default=False),
kms_key_id=dict(),
device_name=dict(),
delete_on_termination=dict(type='bool', default=False),
zone=dict(aliases=['availability_zone', 'aws_zone', 'ec2_zone']),
snapshot=dict(),
state=dict(choices=['absent', 'present', 'list'],
default='present'),
tags=dict(type='dict', default={})))
module = AnsibleModule(argument_spec=argument_spec)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
id = module.params.get('id')
name = module.params.get('name')
instance = module.params.get('instance')
volume_size = module.params.get('volume_size')
encrypted = module.params.get('encrypted')
kms_key_id = module.params.get('kms_key_id')
device_name = module.params.get('device_name')
zone = module.params.get('zone')
snapshot = module.params.get('snapshot')
state = module.params.get('state')
tags = module.params.get('tags')
# Ensure we have the zone or can get the zone
if instance is None and zone is None and state == 'present':
module.fail_json(msg="You must specify either instance or zone")
# Set volume detach flag
if instance == 'None' or instance == '':
instance = None
detach_vol_flag = True
else:
detach_vol_flag = False
# Set changed flag
changed = False
region, ec2_url, aws_connect_params = get_aws_connection_info(module)
if region:
try:
ec2 = connect_to_aws(boto.ec2, region, **aws_connect_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
module.fail_json(msg=str(e))
elif ec2_url:
try:
ec2 = boto.connect_ec2_endpoint(ec2_url, **aws_connect_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError,
boto.provider.ProfileNotFoundError) as e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="Either region or ec2_url must be specified")
if state == 'list':
returned_volumes = []
vols = get_volumes(module, ec2)
for v in vols:
attachment = v.attach_data
returned_volumes.append(get_volume_info(v, state))
module.exit_json(changed=False, volumes=returned_volumes)
if encrypted and not boto_supports_volume_encryption():
module.fail_json(
msg="You must use boto >= v2.29.0 to use encrypted volumes")
if kms_key_id is not None and not boto_supports_kms_key_id():
module.fail_json(msg="You must use boto >= v2.39.0 to use kms_key_id")
# Here we need to get the zone info for the instance. This covers situation where
# instance is specified but zone isn't.
# Useful for playbooks chaining instance launch with volume create + attach and where the
# zone doesn't matter to the user.
inst = None
if instance:
try:
reservation = ec2.get_all_instances(instance_ids=instance)
except BotoServerError as e:
module.fail_json(msg=e.message)
inst = reservation[0].instances[0]
zone = inst.placement
# Check if there is a volume already mounted there.
if device_name:
if device_name in inst.block_device_mapping:
module.exit_json(
msg="Volume mapping for %s already exists on instance %s" %
(device_name, instance),
volume_id=inst.block_device_mapping[device_name].volume_id,
device=device_name,
changed=False)
# Delaying the checks until after the instance check allows us to get volume ids for existing volumes
# without needing to pass an unused volume_size
if not volume_size and not (id or name or snapshot):
module.fail_json(
msg=
"You must specify volume_size or identify an existing volume by id, name, or snapshot"
)
if volume_size and id:
module.fail_json(msg="Cannot specify volume_size together with id")
if state == 'present':
volume, changed = create_volume(module, ec2, zone)
if detach_vol_flag:
volume, changed = detach_volume(module, ec2, volume)
elif inst is not None:
volume, changed = attach_volume(module, ec2, volume, inst)
# Add device, volume_id and volume_type parameters separately to maintain backward compatibility
volume_info = get_volume_info(volume, state)
# deleteOnTermination is not correctly reflected on attachment
if module.params.get('delete_on_termination'):
for attempt in range(0, 8):
if volume_info['attachment_set'].get(
'deleteOnTermination') == 'true':
break
time.sleep(5)
volume = ec2.get_all_volumes(volume_ids=volume.id)[0]
volume_info = get_volume_info(volume, state)
module.exit_json(changed=changed,
volume=volume_info,
device=volume_info['attachment_set']['device'],
volume_id=volume_info['id'],
volume_type=volume_info['type'])
elif state == 'absent':
delete_volume(module, ec2)
def init_connection(log):
conn = boto.connect_ec2_endpoint(log.endpoint_url, log.AK, log.SK)
return conn
def main():
BASE_DIR = "/home/evf/django_evf/"
# this should come from Django later
ec2_access_key = 'svallero'
ec2_secret_key = '5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8'
ssh_key = 'sara'
master_image = 'ami-00001034'
master_flavour = 'm1.small'
master_userdata = 'echo "pippo" > /root/pippo.txt'
worker_flavour = 'm1.large'
worker_userdata = 'echo "pippo" > /root/pippo.txt'
shared_secret = 'pippo'
check_queue_every = 15
min_job_waiting_time = 100
jobs_per_vm = 6
check_vms_every = 45
kill_idle_after = 3600
min_num_workers = 2
max_num_workers = 10
vm_deploy_time = 350
###########################################################################
# substitute variables in template file
#f=open(os.path.join(settings.BASE_DIR, 'context_files/context_master.cloudinit'),'r')
#f_slave=open('/Users/svallero/Django/evf_provisioning/evf/context_files/context_slave_centos.cloudinit','r')
f_slave = open(
os.path.join(BASE_DIR, 'context_files/context_slave_centos.cloudinit'),
'r')
user_data_slave = f_slave.read()
user_data_slave = user_data_slave.replace("<condor_secret>",
str(shared_secret))
# encode workers user-data in base64
user_data_slave_b64 = b64encode(user_data_slave)
#f_master=open('/Users/svallero/Django/evf_provisioning/evf/context_files/context_master_centos.cloudinit','r')
f_master = open(
os.path.join(BASE_DIR,
'context_files/context_master_centos.cloudinit'), 'r')
user_data_master = f_master.read()
user_data_master = user_data_master.replace("<check_queue_every_s>",
str(check_queue_every))
user_data_master = user_data_master.replace("<check_vms_every_s>",
str(check_vms_every))
user_data_master = user_data_master.replace("<check_vms_in_error_every_s>",
str(check_vms_every))
user_data_master = user_data_master.replace("<waiting_jobs_time_s>",
str(min_job_waiting_time))
user_data_master = user_data_master.replace("<n_jobs_per_vm>",
str(jobs_per_vm))
user_data_master = user_data_master.replace("<idle_for_time_s>",
str(kill_idle_after))
user_data_master = user_data_master.replace("<estimated_vm_deploy_time_s>",
str(vm_deploy_time))
user_data_master = user_data_master.replace("<min_vms>",
str(min_num_workers))
user_data_master = user_data_master.replace("<max_vms>",
str(max_num_workers))
user_data_master = user_data_master.replace("<aws_access_key_id>",
str(ec2_access_key))
user_data_master = user_data_master.replace("<aws_secret_access_key>",
str(ec2_secret_key))
user_data_master = user_data_master.replace("<image_id>",
str(master_image))
user_data_master = user_data_master.replace("<key_name>", str(ssh_key))
user_data_master = user_data_master.replace("<flavour>",
str(worker_flavour))
user_data_master = user_data_master.replace("<user_data_b64>",
str(user_data_slave_b64))
user_data_master = user_data_master.replace("<condor_secret>",
str(shared_secret))
#print (user_data_master)
#print user_data_slave_b64
conn = boto.connect_ec2_endpoint("https://one-master.to.infn.it/ec2api/",
aws_access_key_id=ec2_access_key,
aws_secret_access_key=ec2_secret_key,
validate_certs=False)
conn.run_instances(master_image,
instance_type=master_flavour,
user_data=user_data_master)
reservations = conn.get_all_reservations()
inst = reservations[0].instances
return inst[-1]
def run_module():
module_args = dict(
gcloud_url=dict(type='str',
required=False,
default="https://api.thegcloud.com"),
gcloud_access_key=dict(type='str', required=True),
gcloud_secret_key=dict(type='str', required=True),
id=dict(type='int', required=False),
image=dict(type='str', required=False),
instance_type=dict(type='str', required=False),
count=dict(type='int', required=False, default=1),
private_ip=dict(type='bool', required=False, default=False),
state=dict(type='str', required=False, default="present"),
zone=dict(type='str', required=False, default="ord1"),
tag=dict(type='str', required=False),
wait_for=dict(type='int', required=False, default=0),
)
result = dict(changed=False, message='')
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
return result
gcloud_url = module.params['gcloud_url']
gcloud_access_key = module.params['gcloud_access_key']
gcloud_secret_key = module.params['gcloud_secret_key']
if module.params['id']:
id = module.params['id']
else:
id = None
if module.params['image']:
image = module.params['image']
else:
image = None
if module.params['instance_type']:
instance_type = module.params['instance_type']
else:
instance_type = None
count = module.params['count']
private_ip = module.params['private_ip']
state = module.params['state']
zone = module.params['zone']
if module.params['tag']:
tag = module.params['tag']
else:
tag = None
wait_for = module.params['wait_for']
fail_msg = ""
try:
conn = boto.connect_ec2_endpoint(gcloud_url, gcloud_access_key,
gcloud_secret_key)
except Exception as e:
fail_msg = "Unable to connect to region: %s" % gcloud_url
module.fail_json(msg=fail_msg, **result)
try:
if state == "present":
if id and instance_type:
conn.modify_instance_attribute(instance_id=id,
attribute="InstanceType",
value=instance_type)
result['changed'] = True
result['message'] = "VM Action was successful"
module.exit_json(**result)
if not image:
result['message'] = "Image attribute required on create"
module.fail_json(msg=result['message'], **result)
valid = False
for v in conn.get_all_images():
if v.name == image:
valid = True
if not valid:
result['message'] = "Image selected is invalid: %s" % image
module.fail_json(msg=result['message'], **result)
if not instance_type:
result['message'] = "Instance type required on create"
module.fail_json(msg=result['message'], **result)
valid = False
for v in conn.get_all_zones():
if v.name == zone:
valid = True
if not valid:
result['message'] = "Zone selected is invalid: %s" % image
module.fail_json(msg=result['message'], **result)
instances = []
if private_ip == True:
for idx in range(count):
instanceId = conn.run_instances(
image_id=image,
instance_type=instance_type,
placement=zone,
private_ip_address=True)
instances.append({"instance_id": instanceId.instances[0].id, "ip_address": instanceId.instances[0].publicIpAddress, \
"password": conn.get_password_data(instanceId.instances[0].id)})
if tag:
conn.create_tags([instanceId.instances[0].id],
{tag: ''})
else:
for idx in range(count):
instanceId = conn.run_instances(
image_id=image,
instance_type=instance_type,
placement=zone,
private_ip_address=False)
instances.append({"instance_id": instanceId.instances[0].id, "ip_address": instanceId.instances[0].publicIpAddress, \
"password": conn.get_password_data(instanceId.instances[0].id)})
if tag:
conn.create_tags([instanceId.instances[0].id],
{tag: ''})
elapsed_time = 0
start_time = time.time()
while elapsed_time < wait_for:
time.sleep(5)
elapsed_time = time.time() - start_time
break_loop = True
for instance in instances:
status = conn.get_all_instance_status(
instance['instance_id'])[0].state_code
if status != 0:
break_loop = False
if break_loop:
break
if wait_for > 0:
for instance in instances:
status = conn.get_all_instance_status(
instance['instance_id'])[0].state_code
if status != 0:
raise Exception(
"Timeout exceeded on wait_for parameter")
result['instances'] = instances
elif not id or state not in [
'running', 'restarted', 'stopped', 'absent'
]:
result['message'] = "Failed"
if not id:
fail_msg = 'State attribute requires a valid ID'
else:
fail_msg = 'State attribute requires a valid state'
elif state == "running":
conn.start_instances(instance_ids=[id])
elif state == "restarted":
conn.reboot_instances(instance_ids=[id])
elif state == "stopped":
conn.stop_instances(instance_ids=[id])
elif state == "absent":
conn.terminate_instances(instance_ids=[id])
except Exception as e:
fail_msg = e
if fail_msg:
result['message'] = "VM Action was unsuccessful"
module.fail_json(msg=fail_msg, **result)
result['changed'] = True
result['message'] = "VM Action was successful"
module.exit_json(**result)
def ec2_connect(module):
""" Return an ec2 connection"""
region, ec2_url, boto_params = get_aws_connection_info(module)
# If we have a region specified, connect to its endpoint.
if region:
try:
ec2 = connect_to_aws(boto.ec2, region, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e:
module.fail_json(msg=str(e))
# Otherwise, no region so we fallback to the old connection method
elif ec2_url:
try:
ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="Either region or ec2_url must be specified")
return ec2
def paging(pause=0, marker_property='marker'):
""" Adds paging to boto retrieval functions that support a 'marker'
this is configurable as not all boto functions seem to use the
same name.
"""
def wrapper(f):
def page(*args, **kwargs):
results = []
def main(farm):
retval='dummy'
# get farm parameters
ec2_access_key=farm.ec2_access_key
ec2_secret_key=farm.ec2_secret_key
ssh_key=farm.ssh_key
master_image=farm.master_image
master_flavour=farm.master_flavour
master_userdata=farm.master_userdata
worker_flavour=farm.worker_flavour
worker_userdata=farm.worker_userdata
shared_secret=farm.shared_secret
check_queue_every=farm.check_queue_every
min_job_waiting_time=farm.min_job_waiting_time
jobs_per_vm=farm.jobs_per_vm
check_vms_every=farm.check_vms_every
kill_idle_after=farm.kill_idle_after
min_num_workers=farm.min_num_workers
max_num_workers=farm.max_num_workers
vm_deploy_time=farm.vm_deploy_time
#################################
image_description=farm.get_master_image_display()
slave_context='context_files/context_slave_centos.cloudinit'
master_context='context_files/context_master_centos.cloudinit'
if 'CVMFS' in image_description:
slave_context='context_files/context_slave_centos_cvmfs.cloudinit'
master_context='context_files/context_master_centos_cvmfs.cloudinit'
try:
f_slave=open(os.path.join(settings.BASE_DIR, slave_context),'r')
user_data_slave=f_slave.read()
user_data_slave=user_data_slave.replace("<condor_secret>", str(shared_secret))
user_data_slave=user_data_slave.replace("<worker_userdata>", indent(worker_userdata,' '))
# encode workers user-data in base64
user_data_slave_b64=b64encode(user_data_slave.encode('utf-8')).decode('ascii')
except:
retval='ERROR - could not open and edit slave template file!'
return retval
# substitute variables in master template file
try:
f_master=open(os.path.join(settings.BASE_DIR, master_context),'r')
user_data_master=f_master.read()
user_data_master=user_data_master.replace("<check_queue_every_s>", str(check_queue_every))
user_data_master=user_data_master.replace("<check_vms_every_s>", str(check_vms_every))
user_data_master=user_data_master.replace("<check_vms_in_error_every_s>", str(check_vms_every))
user_data_master=user_data_master.replace("<waiting_jobs_time_s>", str(min_job_waiting_time))
user_data_master=user_data_master.replace("<n_jobs_per_vm>", str(jobs_per_vm))
user_data_master=user_data_master.replace("<idle_for_time_s>", str(kill_idle_after))
user_data_master=user_data_master.replace("<estimated_vm_deploy_time_s>", str(vm_deploy_time))
user_data_master=user_data_master.replace("<min_vms>", str(min_num_workers))
user_data_master=user_data_master.replace("<max_vms>", str(max_num_workers))
user_data_master=user_data_master.replace("<aws_access_key_id>", str(ec2_access_key))
user_data_master=user_data_master.replace("<aws_secret_access_key>", str(ec2_secret_key))
user_data_master=user_data_master.replace("<image_id>", str(master_image))
user_data_master=user_data_master.replace("<key_name>", str(ssh_key))
user_data_master=user_data_master.replace("<flavour>", str(worker_flavour))
user_data_master=user_data_master.replace("<user_data_b64>", str(user_data_slave_b64))
user_data_master=user_data_master.replace("<condor_secret>", str(shared_secret))
user_data_master=user_data_master.replace("<master_userdata>", indent(master_userdata,' '))
except:
retval='ERROR - could not open and edit master template file!'
return retval
# make ec2 connection and run instance
try:
conn=boto.connect_ec2_endpoint("https://one-master.to.infn.it/ec2api/",
aws_access_key_id=str(ec2_access_key),
aws_secret_access_key=str(ec2_secret_key),
validate_certs=True,
#is_secure=True,
debug=10)
conn.run_instances(master_image,instance_type=master_flavour,key_name=ssh_key, user_data=str(user_data_master))
reservations = conn.get_all_reservations()
inst=reservations[0].instances
except:
retval='ERROR - could not make ec2 connection and run instance!'
return retval
return inst[-1]
export AWS_ACCESS_KEY=JFIOQNAKEIFJJAKDLIJA
export AWS_SECRET_KEY=3jfioajkle+OnfAEV5OIvj5nLnRy2jfklZRop3nn
"""
sys.exit(1)
try:
ec2_url = "https://%s.ec2.amazonaws.com" % os.environ['EC2_REGION']
except KeyError:
pass
try:
ec2_url = os.environ['EC2_URL']
except KeyError:
pass
ec2_conn = boto.connect_ec2_endpoint(ec2_url, AWS_ACCESS_KEY, AWS_SECRET_KEY)
if ec2_filter:
if re.match("^i-", ec2_filter):
# Instance ID, like: i-581f38e4
ec2_filter_dict = {"instance-id": ec2_filter}
show_all_instances = True # override, since this can only return 1 instance
reservations = ec2_conn.get_all_instances(filters=ec2_filter_dict)
elif re.match("^[a-z][0-9]\......", ec2_filter):
# Instance Type, like m1.small
ec2_filter_dict = {"instance-type": ec2_filter}
reservations = ec2_conn.get_all_instances(filters=ec2_filter_dict)
elif re.match("^10\.(?:[0-9]{1,3}\.){2}[0-9]{1,3}$", ec2_filter):
# Private IP, like 10.45.37.122
ec2_filter_dict = {"private-ip-address": ec2_filter}
show_all_instances = True # override, since this can only return 1 instance
export AWS_SECRET_KEY=3jfioajkle+OnfAEV5OIvj5nLnRy2jfklZRop3nn
"""
sys.exit(1)
try:
ec2_url = "https://%s.ec2.amazonaws.com" % os.environ['EC2_REGION']
except KeyError:
pass
try:
ec2_url = os.environ['EC2_URL']
except KeyError:
pass
ec2_conn = boto.connect_ec2_endpoint(ec2_url, AWS_ACCESS_KEY, AWS_SECRET_KEY)
if ec2_filter:
if re.match( "^i-", ec2_filter ):
# Instance ID, like: i-581f38e4
ec2_filter_dict = { "instance-id": ec2_filter }
show_all_instances = True # override, since this can only return 1 instance
reservations = ec2_conn.get_all_instances( filters=ec2_filter_dict)
elif re.match( "^[a-z][0-9]\......", ec2_filter ):
# Instance Type, like m1.small
ec2_filter_dict = { "instance-type": ec2_filter }
reservations = ec2_conn.get_all_instances(filters=ec2_filter_dict)
elif re.match( "^10\.(?:[0-9]{1,3}\.){2}[0-9]{1,3}$", ec2_filter ):
# Private IP, like 10.45.37.122
ec2_filter_dict = { "private-ip-address": ec2_filter }
show_all_instances = True # override, since this can only return 1 instance
def connect(name) : return boto.connect_ec2_endpoint(profiles[name][0], profiles[name][1], profiles[name][2])
def connect_ec2(args):
"""
Connect to EC2 API by supplied arguments.
Return EC2 connection object.
"""
# Prepare EC2 connection parameters
if not args.ec2_access_key:
if os.getenv('EC2_ACCESS_KEY'):
args.ec2_access_key = os.getenv('EC2_ACCESS_KEY')
elif os.getenv('AWS_ACCESS_KEY'):
args.ec2_access_key = os.getenv('AWS_ACCESS_KEY')
else:
lg.error(
"EC2 Access Key not supplied. Use EC2_ACCESS_KEY or AWS_ACCESS_KEY environment variables or command line option"
)
sys.exit(1)
if not args.ec2_secret_key:
if os.getenv('EC2_SECRET_KEY'):
args.ec2_secret_key = os.getenv('EC2_SECRET_KEY')
elif os.getenv('AWS_SECRET_KEY'):
args.ec2_secret_key = os.getenv('AWS_SECRET_KEY')
else:
lg.error(
"EC2 Secret Key not supplied. Use EC2_SECRET_KEY or AWS_SECRET_KEY environment variables or command line option"
)
sys.exit(1)
if not args.ec2_url:
if os.getenv('EC2_URL'):
args.ec2_url = os.getenv('EC2_URL')
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
if args.timeout:
boto.config.set('Boto', 'http_socket_timeout', str(args.timeout))
if args.cert:
boto.config.set('Boto', 'ca_certificates_file', args.cert)
# Connect to EC2
if args.ec2_url:
# Special connection to EC2-compatible API (eg. OpenStack)
if args.insecure:
validate_certs = False
else:
validate_certs = True
lg.debug("Connecting to EC2: url=%s, validate_certs=%s" %
(args.ec2_url, validate_certs))
ec2 = boto.connect_ec2_endpoint(
url=args.ec2_url,
validate_certs=validate_certs,
aws_access_key_id=args.ec2_access_key,
aws_secret_access_key=args.ec2_secret_key)
else:
# Standard connection to AWS EC2
ec2 = boto.ec2.connect_to_region(
args.ec2_region,
aws_access_key_id=args.ec2_access_key,
aws_secret_access_key=args.ec2_secret_key)
return ec2
def main(farm):
retval = "dummy"
# get farm parameters
ec2_access_key = farm.ec2_access_key
ec2_secret_key = farm.ec2_secret_key
ssh_key = farm.ssh_key
master_image = farm.master_image
master_flavour = farm.master_flavour
master_userdata = farm.master_userdata
worker_flavour = farm.worker_flavour
worker_userdata = farm.worker_userdata
shared_secret = farm.shared_secret
check_queue_every = farm.check_queue_every
min_job_waiting_time = farm.min_job_waiting_time
jobs_per_vm = farm.jobs_per_vm
check_vms_every = farm.check_vms_every
kill_idle_after = farm.kill_idle_after
min_num_workers = farm.min_num_workers
max_num_workers = farm.max_num_workers
vm_deploy_time = farm.vm_deploy_time
#################################
# substitute variables in slave template file
try:
f_slave = open(os.path.join(settings.BASE_DIR, "context_files/context_slave_centos.cloudinit"), "r")
user_data_slave = f_slave.read()
user_data_slave = user_data_slave.replace("<condor_secret>", str(shared_secret))
user_data_slave = user_data_slave.replace("<worker_userdata>", indent(worker_userdata, " "))
# encode workers user-data in base64
user_data_slave_b64 = b64encode(user_data_slave.encode("utf-8")).decode("ascii")
except:
retval = "ERROR - could not open and edit slave template file!"
return retval
# substitute variables in master template file
try:
f_master = open(os.path.join(settings.BASE_DIR, "context_files/context_master_centos.cloudinit"), "r")
user_data_master = f_master.read()
user_data_master = user_data_master.replace("<check_queue_every_s>", str(check_queue_every))
user_data_master = user_data_master.replace("<check_vms_every_s>", str(check_vms_every))
user_data_master = user_data_master.replace("<check_vms_in_error_every_s>", str(check_vms_every))
user_data_master = user_data_master.replace("<waiting_jobs_time_s>", str(min_job_waiting_time))
user_data_master = user_data_master.replace("<n_jobs_per_vm>", str(jobs_per_vm))
user_data_master = user_data_master.replace("<idle_for_time_s>", str(kill_idle_after))
user_data_master = user_data_master.replace("<estimated_vm_deploy_time_s>", str(vm_deploy_time))
user_data_master = user_data_master.replace("<min_vms>", str(min_num_workers))
user_data_master = user_data_master.replace("<max_vms>", str(max_num_workers))
user_data_master = user_data_master.replace("<aws_access_key_id>", str(ec2_access_key))
user_data_master = user_data_master.replace("<aws_secret_access_key>", str(ec2_secret_key))
user_data_master = user_data_master.replace("<image_id>", str(master_image))
user_data_master = user_data_master.replace("<key_name>", str(ssh_key))
user_data_master = user_data_master.replace("<flavour>", str(worker_flavour))
user_data_master = user_data_master.replace("<user_data_b64>", str(user_data_slave_b64))
user_data_master = user_data_master.replace("<condor_secret>", str(shared_secret))
user_data_master = user_data_master.replace("<master_userdata>", indent(master_userdata, " "))
except:
retval = "ERROR - could not open and edit master template file!"
return retval
# make ec2 connection and run instance
try:
conn = boto.connect_ec2_endpoint(
"https://one-master.to.infn.it/ec2api/",
aws_access_key_id=ec2_access_key,
aws_secret_access_key=ec2_secret_key,
)
# validate_certs=False)
conn.run_instances(
master_image, instance_type=master_flavour, key_name=ssh_key, user_data=str(user_data_master)
)
reservations = conn.get_all_reservations()
inst = reservations[0].instances
except:
retval = "ERROR - could not make ec2 connection and run instance!"
return retval
return inst[-1]