Пример #1
0
def make_parallel(action, args, threads, azs, field):
	"""Make parallel requests to CROC Cloud."""

	process_pool = []
	results_queue = Queue()

	connection = boto.connect_ec2_endpoint(os.environ["EC2_URL"])

	def request_maker(queue, args):
		print "Start: {0}".format(datetime.datetime.now())
		try:
			response = connection.make_request(action, args)
			response = response.read()
		except Exception as e:
			response = e

		queue.put((response,))

	try:
		if azs:
			process_pool = [ Process(target=request_maker, args=(results_queue, dict(args, **{field: az})),)
				for i in xrange(0, threads) for az in azs ]
		else:
			process_pool = [ Process(target=request_maker, args=(results_queue, args,))
				for i in xrange(0, threads) ]
		map(lambda p: p.start(), process_pool)
	finally:
		map(lambda p: p.join(), process_pool)

	return [ results_queue.get()[0] for p in process_pool ]
Пример #2
0
def connect_ec2(args):
    """
    Connect to EC2 API by supplied arguments.
    Return EC2 connection object.
    """
    # Prepare EC2 connection parameters
    if not args.ec2_access_key:
        if os.getenv('EC2_ACCESS_KEY'):
            args.ec2_access_key = os.getenv('EC2_ACCESS_KEY')
        elif os.getenv('AWS_ACCESS_KEY'):
            args.ec2_access_key = os.getenv('AWS_ACCESS_KEY')
        else:
            lg.error("EC2 Access Key not supplied. Use EC2_ACCESS_KEY or AWS_ACCESS_KEY environment variables or command line option")
            sys.exit(1)

    if not args.ec2_secret_key:
        if os.getenv('EC2_SECRET_KEY'):
            args.ec2_secret_key = os.getenv('EC2_SECRET_KEY')
        elif os.getenv('AWS_SECRET_KEY'):
            args.ec2_secret_key = os.getenv('AWS_SECRET_KEY')
        else:
            lg.error("EC2 Secret Key not supplied. Use EC2_SECRET_KEY or AWS_SECRET_KEY environment variables or command line option")
            sys.exit(1)

    if not args.ec2_url:
        if os.getenv('EC2_URL'):
            args.ec2_url = os.getenv('EC2_URL')

    if not boto.config.has_section('Boto'):
        boto.config.add_section('Boto')

    if args.timeout:
        boto.config.set('Boto','http_socket_timeout', str(args.timeout))

    if args.cert:
        boto.config.set('Boto', 'ca_certificates_file', args.cert)

    # Connect to EC2
    if args.ec2_url:
        # Special connection to EC2-compatible API (eg. OpenStack)
        if args.insecure:
            validate_certs = False
        else:
            validate_certs = True

        lg.debug("Connecting to EC2: url=%s, validate_certs=%s" % (args.ec2_url, validate_certs))
        ec2 = boto.connect_ec2_endpoint(url=args.ec2_url,
                                validate_certs=validate_certs,
                                aws_access_key_id=args.ec2_access_key,
                                aws_secret_access_key=args.ec2_secret_key)
    else:
        # Standard connection to AWS EC2
        ec2 = boto.ec2.connect_to_region(args.ec2_region,
                                         aws_access_key_id=args.ec2_access_key,
                                         aws_secret_access_key=args.ec2_secret_key)

    return ec2
Пример #3
0
  def _init_ec2(self):
    self.ec2h = boto.connect_ec2_endpoint(
      self.cf['ec2']['api_url'],
      aws_access_key_id=self.cf['ec2']['aws_access_key_id'],
      aws_secret_access_key=self.cf['ec2']['aws_secret_access_key'],
      api_version=self.cf['ec2']['api_version'])

    self.ec2img = self.ec2_image( self.cf['ec2']['image_id'] )
    if self.ec2img is None:
      self.logctl.error('Cannot find EC2 image "%s"', self.cf['ec2']['image_id'])
    else:
      self.logctl.debug('EC2 image "%s" found' % self.cf['ec2']['image_id'])
Пример #4
0
 def __init__(self, configuration=None, access_key=None, secret_key=None, proxy=None, proxy_port=None):
     self.proxy = proxy
     self.proxy_port = proxy_port
     if configuration is not None:
         self.load_configuration(configuration)
     elif access_key is not None and secret_key is not None:
         self.conn = boto.connect_ec2_endpoint(ENDPOINT, access_key, secret_key, proxy=self.proxy,
                                               proxy_port=self.proxy_port)
     elif access_key is not None and secret_key is None:
         raise ValueError("access_key is set but not secret_key")
     elif access_key is None and secret_key is not None:
         raise ValueError("secret_key is set but not access_key")
Пример #5
0
	def test_boto_connect(self):
		conn = boto.connect_ec2_endpoint(config.EC2_URL, aws_access_key_id=config.EC2_ACCESS, aws_secret_access_key=config.EC2_SECRET)

		nets = conn.get_all_addresses()
		images = conn.get_all_images()
		snaps = conn.get_all_snapshots()
		#conn.run_instances(name)

		#r = conn.run_instances(BASE_IMAGE)
		#ins = conn.get_all_instances(instance_ids=[u'i-00000017'])
		instances = conn.get_all_instances()

		#conn.terminate_instances(instance_ids=[u'i-00000016'])
		pass
Пример #6
0
    def load_configuration(self, fp):
        """
        Opens a yaml file and reads from it, verifying that it contains at least the two authentication fields we need.

        :param string fp: Path to the yaml configuration file

        :raises ValueError: When the secret_key and/or access_key are missing in the configuration file.
        """
        with open(fp, 'r') as stream:
            cnf = yaml.load(stream)
            if 'access_key' in cnf and 'secret_key' in cnf:
                self.conn = boto.connect_ec2_endpoint(ENDPOINT, cnf['access_key'], cnf['secret_key'], proxy=self.proxy,
                                                      proxy_port=self.proxy_port)
            else:
                raise ValueError("Need access_key and secret_key in {} configuration file".format(fp))
Пример #7
0
    def create_client(self):
        """Return ec2 client."""
        import boto

        kc = self.keystone()
        if kc.version != "v2.0":
            raise exceptions.RallyException(
                _("Rally EC2 benchmark currently supports only"
                  "Keystone version 2"))
        ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
                                       tenant_id=kc.auth_tenant_id)
        client = boto.connect_ec2_endpoint(
            url=self._get_endpoint(),
            aws_access_key_id=ec2_credential.access,
            aws_secret_access_key=ec2_credential.secret,
            is_secure=self.credential.insecure)
        return client
Пример #8
0
def main():

  ec2_url = os.environ.get('EC2_URL')
  ec2_access_key = os.environ.get('EC2_ACCESS_KEY')
  ec2_secret_key = os.environ.get('EC2_SECRET_KEY')
  ec2_api_version = os.environ.get('EC2_API_VERSION')

  print "Environment:"
  print "EC2_URL: %s" % ec2_url
  print "EC2_ACCESS_KEY: %s" % ec2_access_key
  print "EC2_SECRET_KEY: %s" % ec2_secret_key
  print "EC2_API_VERSION: %s" % ec2_api_version

  if ec2_url is None or ec2_access_key is None or ec2_secret_key is None:
    print "You must set all the proper EC2_* envvars for making it work."
    exit(1)

  # Initialize EC2 connection
  ec2h = boto.connect_ec2_endpoint(
    ec2_url,
    aws_access_key_id=ec2_access_key,
    aws_secret_access_key=ec2_secret_key,
    api_version=ec2_api_version)

  # Try to list VMs and images
  try:

    res = ec2h.get_all_reservations()

    print "\n=== Instances ==="
    for r in res:
      for i in r.instances:
        print "id=%s type=%s name=%s ip=%s key=%s state=%s" % (i.id, i.instance_type, i.public_dns_name, i.private_ip_address, i.key_name, i.state)

    img = ec2h.get_all_images()

    print "\n=== Images ==="
    for im in img:
      print "id=%s name=%s" % (im.id, im.name)


  except Exception, e:
    print "Boto can't talk to EC2: check your credentials"
    exit(2)
Пример #9
0
    def create_client(self):
        """Return ec2 client."""
        LOG.warning("rally.osclient.EC2 is deprecated since Rally 0.10.0.")

        import boto

        kc = self.keystone()

        if kc.version != "v2.0":
            raise exceptions.RallyException(
                "Rally EC2 scenario supports only Keystone version 2")
        ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
                                       tenant_id=kc.auth_tenant_id)
        client = boto.connect_ec2_endpoint(
            url=self._get_endpoint(),
            aws_access_key_id=ec2_credential.access,
            aws_secret_access_key=ec2_credential.secret,
            is_secure=self.credential.https_insecure)
        return client
Пример #10
0
 def ec2(self):
     """Return ec2 client."""
     import boto
     kc = self.keystone()
     if kc.version != "v2.0":
         raise exceptions.RallyException(
             _("Rally EC2 benchmark currently supports only"
               "Keystone version 2"))
     ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
                                    tenant_id=kc.auth_tenant_id)
     ec2_api_url = kc.service_catalog.url_for(
         service_type=consts.ServiceType.EC2,
         endpoint_type=self.endpoint.endpoint_type,
         region_name=self.endpoint.region_name)
     client = boto.connect_ec2_endpoint(
         url=ec2_api_url,
         aws_access_key_id=ec2_credential.access,
         aws_secret_access_key=ec2_credential.secret,
         is_secure=self.endpoint.insecure)
     return client
Пример #11
0
 def create_client(self):
     """Return ec2 client."""
     import boto
     kc = self.keystone()
     if kc.version != "v2.0":
         raise exceptions.RallyException(
             _("Rally EC2 benchmark currently supports only"
               "Keystone version 2"))
     ec2_credential = kc.ec2.create(user_id=kc.auth_user_id,
                                    tenant_id=kc.auth_tenant_id)
     ec2_api_url = kc.service_catalog.url_for(
         service_type=consts.ServiceType.EC2,
         endpoint_type=self.credential.endpoint_type,
         region_name=self.credential.region_name)
     client = boto.connect_ec2_endpoint(
         url=ec2_api_url,
         aws_access_key_id=ec2_credential.access,
         aws_secret_access_key=ec2_credential.secret,
         is_secure=self.credential.insecure)
     return client
Пример #12
0
def ec2_connect(module):
    """ Return an ec2 connection"""

    region, ec2_url, boto_params = get_aws_connection_info(module)

    # If we have a region specified, connect to its endpoint.
    if region:
        try:
            ec2 = connect_to_aws(boto.ec2, region, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
            module.fail_json(msg=str(e))
    # Otherwise, no region so we fallback to the old connection method
    elif ec2_url:
        try:
            ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
            module.fail_json(msg=str(e))
    else:
        module.fail_json(msg="Either region or ec2_url must be specified")

    return ec2
Пример #13
0
def ec2_connect(module):

    """ Return an ec2 connection"""

    region, ec2_url, boto_params = get_aws_connection_info(module)

    # If we have a region specified, connect to its endpoint.
    if region:
        try:
            ec2 = connect_to_aws(boto.ec2, region, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
            module.fail_json(msg=str(e))
    # Otherwise, no region so we fallback to the old connection method
    elif ec2_url:
        try:
            ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
            module.fail_json(msg=str(e))
    else:
        module.fail_json(msg="Either region or ec2_url must be specified")

    return ec2
Пример #14
0
def ec2_connect(module):

    """ Return an ec2 connection"""

    region, ec2_url, boto_params = get_aws_connection_info(module)

    # If ec2_url is present use it
    if ec2_url:
        try:
            ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
            module.fail_json(msg=str(e))
    # Otherwise, if we have a region specified, connect to its endpoint.
    elif region:
        try:
            ec2 = connect_to_aws(boto.ec2, region, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
            module.fail_json(msg=str(e))
    else:
        module.fail_json(msg="Either region or ec2_url must be specified")

    return ec2
Пример #15
0
def isolated_connection_factory():
	return boto.connect_ec2_endpoint(config.EC2_URL, aws_secret_access_key=config.EC2_SECRET,
	                                 aws_access_key_id=config.EC2_ACCESS)
Пример #16
0
def isolated_connection_factory():
    return boto.connect_ec2_endpoint(config.EC2_URL,
                                     aws_secret_access_key=config.EC2_SECRET,
                                     aws_access_key_id=config.EC2_ACCESS)
Пример #17
0
def main():
    argument_spec = ec2_argument_spec()
    argument_spec.update(
        dict(instance=dict(),
             id=dict(),
             name=dict(),
             volume_size=dict(type='int'),
             volume_type=dict(choices=['standard', 'gp2', 'io1', 'st1', 'sc1'],
                              default='standard'),
             iops=dict(type='int'),
             encrypted=dict(type='bool', default=False),
             kms_key_id=dict(),
             device_name=dict(),
             delete_on_termination=dict(type='bool', default=False),
             zone=dict(aliases=['availability_zone', 'aws_zone', 'ec2_zone']),
             snapshot=dict(),
             state=dict(choices=['absent', 'present', 'list'],
                        default='present'),
             tags=dict(type='dict', default={})))
    module = AnsibleModule(argument_spec=argument_spec)

    if not HAS_BOTO:
        module.fail_json(msg='boto required for this module')

    id = module.params.get('id')
    name = module.params.get('name')
    instance = module.params.get('instance')
    volume_size = module.params.get('volume_size')
    encrypted = module.params.get('encrypted')
    kms_key_id = module.params.get('kms_key_id')
    device_name = module.params.get('device_name')
    zone = module.params.get('zone')
    snapshot = module.params.get('snapshot')
    state = module.params.get('state')
    tags = module.params.get('tags')

    # Ensure we have the zone or can get the zone
    if instance is None and zone is None and state == 'present':
        module.fail_json(msg="You must specify either instance or zone")

    # Set volume detach flag
    if instance == 'None' or instance == '':
        instance = None
        detach_vol_flag = True
    else:
        detach_vol_flag = False

    # Set changed flag
    changed = False

    region, ec2_url, aws_connect_params = get_aws_connection_info(module)

    if region:
        try:
            ec2 = connect_to_aws(boto.ec2, region, **aws_connect_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
            module.fail_json(msg=str(e))
    elif ec2_url:
        try:
            ec2 = boto.connect_ec2_endpoint(ec2_url, **aws_connect_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError,
                boto.provider.ProfileNotFoundError) as e:
            module.fail_json(msg=str(e))
    else:
        module.fail_json(msg="Either region or ec2_url must be specified")

    if state == 'list':
        returned_volumes = []
        vols = get_volumes(module, ec2)

        for v in vols:
            attachment = v.attach_data

            returned_volumes.append(get_volume_info(v, state))

        module.exit_json(changed=False, volumes=returned_volumes)

    if encrypted and not boto_supports_volume_encryption():
        module.fail_json(
            msg="You must use boto >= v2.29.0 to use encrypted volumes")

    if kms_key_id is not None and not boto_supports_kms_key_id():
        module.fail_json(msg="You must use boto >= v2.39.0 to use kms_key_id")

    # Here we need to get the zone info for the instance. This covers situation where
    # instance is specified but zone isn't.
    # Useful for playbooks chaining instance launch with volume create + attach and where the
    # zone doesn't matter to the user.
    inst = None
    if instance:
        try:
            reservation = ec2.get_all_instances(instance_ids=instance)
        except BotoServerError as e:
            module.fail_json(msg=e.message)
        inst = reservation[0].instances[0]
        zone = inst.placement

        # Check if there is a volume already mounted there.
        if device_name:
            if device_name in inst.block_device_mapping:
                module.exit_json(
                    msg="Volume mapping for %s already exists on instance %s" %
                    (device_name, instance),
                    volume_id=inst.block_device_mapping[device_name].volume_id,
                    device=device_name,
                    changed=False)

    # Delaying the checks until after the instance check allows us to get volume ids for existing volumes
    # without needing to pass an unused volume_size
    if not volume_size and not (id or name or snapshot):
        module.fail_json(
            msg=
            "You must specify volume_size or identify an existing volume by id, name, or snapshot"
        )

    if volume_size and id:
        module.fail_json(msg="Cannot specify volume_size together with id")

    if state == 'present':
        volume, changed = create_volume(module, ec2, zone)
        if detach_vol_flag:
            volume, changed = detach_volume(module, ec2, volume)
        elif inst is not None:
            volume, changed = attach_volume(module, ec2, volume, inst)

        # Add device, volume_id and volume_type parameters separately to maintain backward compatibility
        volume_info = get_volume_info(volume, state)

        # deleteOnTermination is not correctly reflected on attachment
        if module.params.get('delete_on_termination'):
            for attempt in range(0, 8):
                if volume_info['attachment_set'].get(
                        'deleteOnTermination') == 'true':
                    break
                time.sleep(5)
                volume = ec2.get_all_volumes(volume_ids=volume.id)[0]
                volume_info = get_volume_info(volume, state)
        module.exit_json(changed=changed,
                         volume=volume_info,
                         device=volume_info['attachment_set']['device'],
                         volume_id=volume_info['id'],
                         volume_type=volume_info['type'])
    elif state == 'absent':
        delete_volume(module, ec2)
Пример #18
0
def init_connection(log):
    conn = boto.connect_ec2_endpoint(log.endpoint_url, log.AK, log.SK)
    return conn
Пример #19
0
def main():
    BASE_DIR = "/home/evf/django_evf/"
    # this should come from Django later
    ec2_access_key = 'svallero'
    ec2_secret_key = '5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8'
    ssh_key = 'sara'
    master_image = 'ami-00001034'
    master_flavour = 'm1.small'
    master_userdata = 'echo "pippo" > /root/pippo.txt'
    worker_flavour = 'm1.large'
    worker_userdata = 'echo "pippo" > /root/pippo.txt'
    shared_secret = 'pippo'
    check_queue_every = 15
    min_job_waiting_time = 100
    jobs_per_vm = 6
    check_vms_every = 45
    kill_idle_after = 3600
    min_num_workers = 2
    max_num_workers = 10
    vm_deploy_time = 350
    ###########################################################################

    # substitute variables in template file
    #f=open(os.path.join(settings.BASE_DIR, 'context_files/context_master.cloudinit'),'r')
    #f_slave=open('/Users/svallero/Django/evf_provisioning/evf/context_files/context_slave_centos.cloudinit','r')
    f_slave = open(
        os.path.join(BASE_DIR, 'context_files/context_slave_centos.cloudinit'),
        'r')
    user_data_slave = f_slave.read()
    user_data_slave = user_data_slave.replace("<condor_secret>",
                                              str(shared_secret))
    # encode workers user-data in base64
    user_data_slave_b64 = b64encode(user_data_slave)

    #f_master=open('/Users/svallero/Django/evf_provisioning/evf/context_files/context_master_centos.cloudinit','r')
    f_master = open(
        os.path.join(BASE_DIR,
                     'context_files/context_master_centos.cloudinit'), 'r')
    user_data_master = f_master.read()

    user_data_master = user_data_master.replace("<check_queue_every_s>",
                                                str(check_queue_every))
    user_data_master = user_data_master.replace("<check_vms_every_s>",
                                                str(check_vms_every))
    user_data_master = user_data_master.replace("<check_vms_in_error_every_s>",
                                                str(check_vms_every))
    user_data_master = user_data_master.replace("<waiting_jobs_time_s>",
                                                str(min_job_waiting_time))
    user_data_master = user_data_master.replace("<n_jobs_per_vm>",
                                                str(jobs_per_vm))
    user_data_master = user_data_master.replace("<idle_for_time_s>",
                                                str(kill_idle_after))
    user_data_master = user_data_master.replace("<estimated_vm_deploy_time_s>",
                                                str(vm_deploy_time))
    user_data_master = user_data_master.replace("<min_vms>",
                                                str(min_num_workers))
    user_data_master = user_data_master.replace("<max_vms>",
                                                str(max_num_workers))
    user_data_master = user_data_master.replace("<aws_access_key_id>",
                                                str(ec2_access_key))
    user_data_master = user_data_master.replace("<aws_secret_access_key>",
                                                str(ec2_secret_key))
    user_data_master = user_data_master.replace("<image_id>",
                                                str(master_image))
    user_data_master = user_data_master.replace("<key_name>", str(ssh_key))
    user_data_master = user_data_master.replace("<flavour>",
                                                str(worker_flavour))
    user_data_master = user_data_master.replace("<user_data_b64>",
                                                str(user_data_slave_b64))
    user_data_master = user_data_master.replace("<condor_secret>",
                                                str(shared_secret))

    #print (user_data_master)
    #print user_data_slave_b64

    conn = boto.connect_ec2_endpoint("https://one-master.to.infn.it/ec2api/",
                                     aws_access_key_id=ec2_access_key,
                                     aws_secret_access_key=ec2_secret_key,
                                     validate_certs=False)
    conn.run_instances(master_image,
                       instance_type=master_flavour,
                       user_data=user_data_master)
    reservations = conn.get_all_reservations()
    inst = reservations[0].instances
    return inst[-1]
Пример #20
0
def run_module():

    module_args = dict(
        gcloud_url=dict(type='str',
                        required=False,
                        default="https://api.thegcloud.com"),
        gcloud_access_key=dict(type='str', required=True),
        gcloud_secret_key=dict(type='str', required=True),
        id=dict(type='int', required=False),
        image=dict(type='str', required=False),
        instance_type=dict(type='str', required=False),
        count=dict(type='int', required=False, default=1),
        private_ip=dict(type='bool', required=False, default=False),
        state=dict(type='str', required=False, default="present"),
        zone=dict(type='str', required=False, default="ord1"),
        tag=dict(type='str', required=False),
        wait_for=dict(type='int', required=False, default=0),
    )

    result = dict(changed=False, message='')

    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)

    if module.check_mode:
        return result

    gcloud_url = module.params['gcloud_url']
    gcloud_access_key = module.params['gcloud_access_key']
    gcloud_secret_key = module.params['gcloud_secret_key']

    if module.params['id']:
        id = module.params['id']
    else:
        id = None

    if module.params['image']:
        image = module.params['image']
    else:
        image = None

    if module.params['instance_type']:
        instance_type = module.params['instance_type']
    else:
        instance_type = None

    count = module.params['count']
    private_ip = module.params['private_ip']
    state = module.params['state']
    zone = module.params['zone']

    if module.params['tag']:
        tag = module.params['tag']
    else:
        tag = None

    wait_for = module.params['wait_for']

    fail_msg = ""
    try:
        conn = boto.connect_ec2_endpoint(gcloud_url, gcloud_access_key,
                                         gcloud_secret_key)
    except Exception as e:
        fail_msg = "Unable to connect to region: %s" % gcloud_url
        module.fail_json(msg=fail_msg, **result)

    try:
        if state == "present":
            if id and instance_type:
                conn.modify_instance_attribute(instance_id=id,
                                               attribute="InstanceType",
                                               value=instance_type)
                result['changed'] = True
                result['message'] = "VM Action was successful"
                module.exit_json(**result)

            if not image:
                result['message'] = "Image attribute required on create"
                module.fail_json(msg=result['message'], **result)

            valid = False
            for v in conn.get_all_images():
                if v.name == image:
                    valid = True

            if not valid:
                result['message'] = "Image selected is invalid: %s" % image
                module.fail_json(msg=result['message'], **result)

            if not instance_type:
                result['message'] = "Instance type required on create"
                module.fail_json(msg=result['message'], **result)

            valid = False
            for v in conn.get_all_zones():
                if v.name == zone:
                    valid = True

            if not valid:
                result['message'] = "Zone selected is invalid: %s" % image
                module.fail_json(msg=result['message'], **result)

            instances = []
            if private_ip == True:
                for idx in range(count):
                    instanceId = conn.run_instances(
                        image_id=image,
                        instance_type=instance_type,
                        placement=zone,
                        private_ip_address=True)
                    instances.append({"instance_id": instanceId.instances[0].id, "ip_address": instanceId.instances[0].publicIpAddress, \
                        "password": conn.get_password_data(instanceId.instances[0].id)})
                    if tag:
                        conn.create_tags([instanceId.instances[0].id],
                                         {tag: ''})
            else:
                for idx in range(count):
                    instanceId = conn.run_instances(
                        image_id=image,
                        instance_type=instance_type,
                        placement=zone,
                        private_ip_address=False)
                    instances.append({"instance_id": instanceId.instances[0].id, "ip_address": instanceId.instances[0].publicIpAddress, \
                        "password": conn.get_password_data(instanceId.instances[0].id)})
                    if tag:
                        conn.create_tags([instanceId.instances[0].id],
                                         {tag: ''})

            elapsed_time = 0
            start_time = time.time()
            while elapsed_time < wait_for:
                time.sleep(5)
                elapsed_time = time.time() - start_time
                break_loop = True
                for instance in instances:
                    status = conn.get_all_instance_status(
                        instance['instance_id'])[0].state_code
                    if status != 0:
                        break_loop = False
                if break_loop:
                    break

            if wait_for > 0:
                for instance in instances:
                    status = conn.get_all_instance_status(
                        instance['instance_id'])[0].state_code
                    if status != 0:
                        raise Exception(
                            "Timeout exceeded on wait_for parameter")

            result['instances'] = instances

        elif not id or state not in [
                'running', 'restarted', 'stopped', 'absent'
        ]:
            result['message'] = "Failed"
            if not id:
                fail_msg = 'State attribute requires a valid ID'
            else:
                fail_msg = 'State attribute requires a valid state'
        elif state == "running":
            conn.start_instances(instance_ids=[id])
        elif state == "restarted":
            conn.reboot_instances(instance_ids=[id])
        elif state == "stopped":
            conn.stop_instances(instance_ids=[id])
        elif state == "absent":
            conn.terminate_instances(instance_ids=[id])
    except Exception as e:
        fail_msg = e

    if fail_msg:
        result['message'] = "VM Action was unsuccessful"
        module.fail_json(msg=fail_msg, **result)

    result['changed'] = True
    result['message'] = "VM Action was successful"

    module.exit_json(**result)
Пример #21
0
def ec2_connect(module):

    """ Return an ec2 connection"""

    region, ec2_url, boto_params = get_aws_connection_info(module)

    # If we have a region specified, connect to its endpoint.
    if region:
        try:
            ec2 = connect_to_aws(boto.ec2, region, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e:
            module.fail_json(msg=str(e))
    # Otherwise, no region so we fallback to the old connection method
    elif ec2_url:
        try:
            ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError), e:
            module.fail_json(msg=str(e))
    else:
        module.fail_json(msg="Either region or ec2_url must be specified")

    return ec2

def paging(pause=0, marker_property='marker'):
    """ Adds paging to boto retrieval functions that support a 'marker'
        this is configurable as not all boto functions seem to use the
        same name.
    """
    def wrapper(f):
        def page(*args, **kwargs):
            results = []
Пример #22
0
def main(farm):
  retval='dummy'
  # get farm parameters
  ec2_access_key=farm.ec2_access_key
  ec2_secret_key=farm.ec2_secret_key
  ssh_key=farm.ssh_key
  master_image=farm.master_image
  master_flavour=farm.master_flavour
  master_userdata=farm.master_userdata 
  worker_flavour=farm.worker_flavour
  worker_userdata=farm.worker_userdata 
  shared_secret=farm.shared_secret
  check_queue_every=farm.check_queue_every
  min_job_waiting_time=farm.min_job_waiting_time
  jobs_per_vm=farm.jobs_per_vm
  check_vms_every=farm.check_vms_every
  kill_idle_after=farm.kill_idle_after
  min_num_workers=farm.min_num_workers
  max_num_workers=farm.max_num_workers
  vm_deploy_time=farm.vm_deploy_time
  #################################

  image_description=farm.get_master_image_display()
  slave_context='context_files/context_slave_centos.cloudinit'
  master_context='context_files/context_master_centos.cloudinit'

  if 'CVMFS' in image_description:
     slave_context='context_files/context_slave_centos_cvmfs.cloudinit'
     master_context='context_files/context_master_centos_cvmfs.cloudinit'
  
  try:
     f_slave=open(os.path.join(settings.BASE_DIR, slave_context),'r')
     user_data_slave=f_slave.read()
     user_data_slave=user_data_slave.replace("<condor_secret>", str(shared_secret))
     user_data_slave=user_data_slave.replace("<worker_userdata>", indent(worker_userdata,'     '))
     # encode workers user-data in base64 
     user_data_slave_b64=b64encode(user_data_slave.encode('utf-8')).decode('ascii')
  except:
     retval='ERROR - could not open and edit slave template file!' 
     return retval
  
  # substitute variables in master template file
  try:
     f_master=open(os.path.join(settings.BASE_DIR, master_context),'r')
     user_data_master=f_master.read()
 
     user_data_master=user_data_master.replace("<check_queue_every_s>", str(check_queue_every))
     user_data_master=user_data_master.replace("<check_vms_every_s>", str(check_vms_every))
     user_data_master=user_data_master.replace("<check_vms_in_error_every_s>", str(check_vms_every))
     user_data_master=user_data_master.replace("<waiting_jobs_time_s>", str(min_job_waiting_time))
     user_data_master=user_data_master.replace("<n_jobs_per_vm>", str(jobs_per_vm))
     user_data_master=user_data_master.replace("<idle_for_time_s>", str(kill_idle_after))
     user_data_master=user_data_master.replace("<estimated_vm_deploy_time_s>", str(vm_deploy_time))
     user_data_master=user_data_master.replace("<min_vms>", str(min_num_workers))
     user_data_master=user_data_master.replace("<max_vms>", str(max_num_workers))
     user_data_master=user_data_master.replace("<aws_access_key_id>", str(ec2_access_key))
     user_data_master=user_data_master.replace("<aws_secret_access_key>", str(ec2_secret_key))
     user_data_master=user_data_master.replace("<image_id>", str(master_image))
     user_data_master=user_data_master.replace("<key_name>", str(ssh_key))
     user_data_master=user_data_master.replace("<flavour>", str(worker_flavour))
     user_data_master=user_data_master.replace("<user_data_b64>", str(user_data_slave_b64))
     user_data_master=user_data_master.replace("<condor_secret>", str(shared_secret))
     user_data_master=user_data_master.replace("<master_userdata>", indent(master_userdata,'     '))
  except:
     retval='ERROR - could not open and edit master template file!' 
     return retval


  # make ec2 connection and run instance
  try:
     conn=boto.connect_ec2_endpoint("https://one-master.to.infn.it/ec2api/",
                                 aws_access_key_id=str(ec2_access_key),
                                 aws_secret_access_key=str(ec2_secret_key),
                                 validate_certs=True,
                                 #is_secure=True,
                                 debug=10)
     conn.run_instances(master_image,instance_type=master_flavour,key_name=ssh_key, user_data=str(user_data_master))
     reservations = conn.get_all_reservations()
     inst=reservations[0].instances
  except:
     retval='ERROR - could not make ec2 connection and run instance!' 
     return retval

  return inst[-1]
Пример #23
0
  export AWS_ACCESS_KEY=JFIOQNAKEIFJJAKDLIJA
  export AWS_SECRET_KEY=3jfioajkle+OnfAEV5OIvj5nLnRy2jfklZRop3nn
"""
        sys.exit(1)

try:
    ec2_url = "https://%s.ec2.amazonaws.com" % os.environ['EC2_REGION']
except KeyError:
    pass

try:
    ec2_url = os.environ['EC2_URL']
except KeyError:
    pass

ec2_conn = boto.connect_ec2_endpoint(ec2_url, AWS_ACCESS_KEY, AWS_SECRET_KEY)

if ec2_filter:
    if re.match("^i-", ec2_filter):
        # Instance ID, like: i-581f38e4
        ec2_filter_dict = {"instance-id": ec2_filter}
        show_all_instances = True  # override, since this can only return 1 instance
        reservations = ec2_conn.get_all_instances(filters=ec2_filter_dict)
    elif re.match("^[a-z][0-9]\......", ec2_filter):
        # Instance Type, like m1.small
        ec2_filter_dict = {"instance-type": ec2_filter}
        reservations = ec2_conn.get_all_instances(filters=ec2_filter_dict)
    elif re.match("^10\.(?:[0-9]{1,3}\.){2}[0-9]{1,3}$", ec2_filter):
        # Private IP, like 10.45.37.122
        ec2_filter_dict = {"private-ip-address": ec2_filter}
        show_all_instances = True  # override, since this can only return 1 instance
Пример #24
0
  export AWS_SECRET_KEY=3jfioajkle+OnfAEV5OIvj5nLnRy2jfklZRop3nn
"""
    sys.exit(1)


try:
  ec2_url = "https://%s.ec2.amazonaws.com" % os.environ['EC2_REGION']
except KeyError:
  pass

try:
  ec2_url = os.environ['EC2_URL']
except KeyError:
  pass

ec2_conn = boto.connect_ec2_endpoint(ec2_url, AWS_ACCESS_KEY, AWS_SECRET_KEY)

if ec2_filter:
  if re.match( "^i-", ec2_filter ):
    # Instance ID, like: i-581f38e4
    ec2_filter_dict = { "instance-id": ec2_filter }
    show_all_instances = True # override, since this can only return 1 instance
    reservations = ec2_conn.get_all_instances( filters=ec2_filter_dict)
  elif re.match( "^[a-z][0-9]\......", ec2_filter ):
    # Instance Type, like m1.small
    ec2_filter_dict = { "instance-type": ec2_filter }
    reservations = ec2_conn.get_all_instances(filters=ec2_filter_dict)
  elif re.match( "^10\.(?:[0-9]{1,3}\.){2}[0-9]{1,3}$", ec2_filter ):
    # Private IP, like 10.45.37.122
    ec2_filter_dict = { "private-ip-address": ec2_filter }
    show_all_instances = True # override, since this can only return 1 instance
Пример #25
0
def connect(name) :
	return boto.connect_ec2_endpoint(profiles[name][0], profiles[name][1], profiles[name][2])
Пример #26
0
def connect_ec2(args):
    """
    Connect to EC2 API by supplied arguments.
    Return EC2 connection object.
    """
    # Prepare EC2 connection parameters
    if not args.ec2_access_key:
        if os.getenv('EC2_ACCESS_KEY'):
            args.ec2_access_key = os.getenv('EC2_ACCESS_KEY')
        elif os.getenv('AWS_ACCESS_KEY'):
            args.ec2_access_key = os.getenv('AWS_ACCESS_KEY')
        else:
            lg.error(
                "EC2 Access Key not supplied. Use EC2_ACCESS_KEY or AWS_ACCESS_KEY environment variables or command line option"
            )
            sys.exit(1)

    if not args.ec2_secret_key:
        if os.getenv('EC2_SECRET_KEY'):
            args.ec2_secret_key = os.getenv('EC2_SECRET_KEY')
        elif os.getenv('AWS_SECRET_KEY'):
            args.ec2_secret_key = os.getenv('AWS_SECRET_KEY')
        else:
            lg.error(
                "EC2 Secret Key not supplied. Use EC2_SECRET_KEY or AWS_SECRET_KEY environment variables or command line option"
            )
            sys.exit(1)

    if not args.ec2_url:
        if os.getenv('EC2_URL'):
            args.ec2_url = os.getenv('EC2_URL')

    if not boto.config.has_section('Boto'):
        boto.config.add_section('Boto')

    if args.timeout:
        boto.config.set('Boto', 'http_socket_timeout', str(args.timeout))

    if args.cert:
        boto.config.set('Boto', 'ca_certificates_file', args.cert)

    # Connect to EC2
    if args.ec2_url:
        # Special connection to EC2-compatible API (eg. OpenStack)
        if args.insecure:
            validate_certs = False
        else:
            validate_certs = True

        lg.debug("Connecting to EC2: url=%s, validate_certs=%s" %
                 (args.ec2_url, validate_certs))
        ec2 = boto.connect_ec2_endpoint(
            url=args.ec2_url,
            validate_certs=validate_certs,
            aws_access_key_id=args.ec2_access_key,
            aws_secret_access_key=args.ec2_secret_key)
    else:
        # Standard connection to AWS EC2
        ec2 = boto.ec2.connect_to_region(
            args.ec2_region,
            aws_access_key_id=args.ec2_access_key,
            aws_secret_access_key=args.ec2_secret_key)

    return ec2
Пример #27
0
def main(farm):
    retval = "dummy"
    # get farm parameters
    ec2_access_key = farm.ec2_access_key
    ec2_secret_key = farm.ec2_secret_key
    ssh_key = farm.ssh_key
    master_image = farm.master_image
    master_flavour = farm.master_flavour
    master_userdata = farm.master_userdata
    worker_flavour = farm.worker_flavour
    worker_userdata = farm.worker_userdata
    shared_secret = farm.shared_secret
    check_queue_every = farm.check_queue_every
    min_job_waiting_time = farm.min_job_waiting_time
    jobs_per_vm = farm.jobs_per_vm
    check_vms_every = farm.check_vms_every
    kill_idle_after = farm.kill_idle_after
    min_num_workers = farm.min_num_workers
    max_num_workers = farm.max_num_workers
    vm_deploy_time = farm.vm_deploy_time
    #################################

    # substitute variables in slave template file
    try:
        f_slave = open(os.path.join(settings.BASE_DIR, "context_files/context_slave_centos.cloudinit"), "r")
        user_data_slave = f_slave.read()
        user_data_slave = user_data_slave.replace("<condor_secret>", str(shared_secret))
        user_data_slave = user_data_slave.replace("<worker_userdata>", indent(worker_userdata, "     "))
        # encode workers user-data in base64
        user_data_slave_b64 = b64encode(user_data_slave.encode("utf-8")).decode("ascii")
    except:
        retval = "ERROR - could not open and edit slave template file!"
        return retval

    # substitute variables in master template file
    try:
        f_master = open(os.path.join(settings.BASE_DIR, "context_files/context_master_centos.cloudinit"), "r")
        user_data_master = f_master.read()

        user_data_master = user_data_master.replace("<check_queue_every_s>", str(check_queue_every))
        user_data_master = user_data_master.replace("<check_vms_every_s>", str(check_vms_every))
        user_data_master = user_data_master.replace("<check_vms_in_error_every_s>", str(check_vms_every))
        user_data_master = user_data_master.replace("<waiting_jobs_time_s>", str(min_job_waiting_time))
        user_data_master = user_data_master.replace("<n_jobs_per_vm>", str(jobs_per_vm))
        user_data_master = user_data_master.replace("<idle_for_time_s>", str(kill_idle_after))
        user_data_master = user_data_master.replace("<estimated_vm_deploy_time_s>", str(vm_deploy_time))
        user_data_master = user_data_master.replace("<min_vms>", str(min_num_workers))
        user_data_master = user_data_master.replace("<max_vms>", str(max_num_workers))
        user_data_master = user_data_master.replace("<aws_access_key_id>", str(ec2_access_key))
        user_data_master = user_data_master.replace("<aws_secret_access_key>", str(ec2_secret_key))
        user_data_master = user_data_master.replace("<image_id>", str(master_image))
        user_data_master = user_data_master.replace("<key_name>", str(ssh_key))
        user_data_master = user_data_master.replace("<flavour>", str(worker_flavour))
        user_data_master = user_data_master.replace("<user_data_b64>", str(user_data_slave_b64))
        user_data_master = user_data_master.replace("<condor_secret>", str(shared_secret))
        user_data_master = user_data_master.replace("<master_userdata>", indent(master_userdata, "     "))
    except:
        retval = "ERROR - could not open and edit master template file!"
        return retval

    # make ec2 connection and run instance
    try:
        conn = boto.connect_ec2_endpoint(
            "https://one-master.to.infn.it/ec2api/",
            aws_access_key_id=ec2_access_key,
            aws_secret_access_key=ec2_secret_key,
        )
        # validate_certs=False)
        conn.run_instances(
            master_image, instance_type=master_flavour, key_name=ssh_key, user_data=str(user_data_master)
        )
        reservations = conn.get_all_reservations()
        inst = reservations[0].instances
    except:
        retval = "ERROR - could not make ec2 connection and run instance!"
        return retval

    return inst[-1]