def download_ssh_pubkey(self, ec2_keypair):
try:
bucket = self.s3.get_bucket(self.s3_bucket_name)
s3_entry = S3Key(bucket)
s3_entry.key = self.ssh_pubkey_s3_key_prefix + ec2_keypair.fingerprint
ssh_pubkey = s3_entry.get_contents_as_string()
except S3ResponseError as e:
if e.status == 404:
raise UserError(
"There is no matching SSH pub key stored in S3 for EC2 key pair %s. Has "
"it been registered, e.g using the cgcloud's register-key command?"
% ec2_keypair.name)
else:
raise
fingerprint_len = len(ec2_keypair.fingerprint.split(':'))
if fingerprint_len == 20: # 160 bit SHA-1
# The fingerprint is that of a private key. We can't get at the private key so we
# can't verify the public key either. So this is inherently insecure. However,
# remember that the only reason why we are dealing with n EC2-generated private
# key is that the Jenkins' EC2 plugin expects a 20 byte fingerprint. See
# https://issues.jenkins-ci.org/browse/JENKINS-20142 for details. Once that issue
# is fixed, we can switch back to just using imported keys and 16-byte fingerprints.
pass
elif fingerprint_len == 16: # 128 bit MD5
fingerprint = ec2_keypair_fingerprint(ssh_pubkey)
if ec2_keypair.fingerprint != fingerprint:
raise UserError(
"Fingerprint mismatch for key %s! Expected %s but got %s. The EC2 keypair "
"doesn't match the public key stored in S3." %
(ec2_keypair.name, ec2_keypair.fingerprint, fingerprint))
return ssh_pubkey
def __assert_attachable( self ):
if self.volume.status != 'available':
raise UserError( "EBS volume %s is not available." % self.name )
expected_zone = self.availability_zone
if self.volume.zone != expected_zone:
raise UserError( "Availability zone of EBS volume %s is %s but should be %s."
% (self.name, self.volume.zone, expected_zone) )
def __get_master_pubkey( self ):
ec2_keypair_name = JenkinsMaster.ec2_keypair_name( self.ctx )
ec2_keypair = self.ctx.ec2.get_key_pair( ec2_keypair_name )
if ec2_keypair is None:
raise UserError( "Missing EC2 keypair named '%s'. You must create the master before "
"creating slaves." % ec2_keypair_name )
return self.ctx.download_ssh_pubkey( ec2_keypair )
def run(self, options):
if options.share_path is not None:
if not os.path.exists(options.share_path):
raise UserError("No such file or directory: '%s'" %
options.share_path)
# --leader-instance-type should default to the value of --instance-type
if options.instance_type is None:
options.instance_type = options.worker_instance_type
super(CreateClusterCommand, self).run(options)
def run_in_ctx(self, options, ctx):
with open(options.ssh_public_key) as f:
ssh_public_key = f.read()
try:
ctx.register_ssh_pubkey(ec2_keypair_name=ctx.resolve_me(
options.ec2_keypair_name, drop_hostname=False),
ssh_pubkey=ssh_public_key,
force=options.force)
except ValueError as e:
raise UserError(cause=e)
def attach(self, instance_id, device):
if self.volume.attach_data.instance_id == instance_id:
log.info("Volume '%s' already attached to instance '%s'." %
(self.volume.id, instance_id))
else:
self.__assert_attachable()
self.ec2.attach_volume(volume_id=self.volume.id,
instance_id=instance_id,
device=device)
self.__wait_transition(self.volume, {'available'}, 'in-use')
if self.volume.attach_data.instance_id != instance_id:
raise UserError("Volume %s is not attached to this instance.")
def __lookup(self):
"""
Ensure that an EBS volume of the given name is available in the current availability zone.
If the EBS volume exists but has been placed into a different zone, or if it is not
available, an exception will be thrown.
:rtype: boto.ec2.volume.Volume
"""
volumes = self.ec2.get_all_volumes(filters={'tag:Name': self.name})
if len(volumes) < 1:
return None
if len(volumes) > 1:
raise UserError("More than one EBS volume named %s" % self.name)
return volumes[0]
def register_slaves(self, slave_clss, clean=False, instance_type=None):
with self.__patch_jenkins_config() as config:
templates = config.find('.//hudson.plugins.ec2.EC2Cloud/templates')
if templates is None:
raise UserError(
"Can't find any configuration for the Jenkins Amazon EC2 plugin. Make sure it is "
"installed and configured on the %s in %s." %
(self.role(), self.ctx.namespace))
template_element_name = 'hudson.plugins.ec2.SlaveTemplate'
if clean:
for old_template in templates.findall(template_element_name):
templates.getchildren().remove(old_template)
for slave_cls in slave_clss:
slave = slave_cls(self.ctx)
images = slave.list_images()
try:
image = images[-1]
except IndexError:
raise UserError("No images for '%s'" % slave_cls.role())
new_template = slave.slave_config_template(
image, instance_type)
description = new_template.find('description').text
found = False
for old_template in templates.findall(template_element_name):
if old_template.find('description').text == description:
if found:
raise RuntimeError(
'More than one existing slave definition for %s. '
'Fix and try again' % description)
i = templates.getchildren().index(old_template)
templates[i] = new_template
found = True
if not found:
templates.append(new_template)
# newer versions of Jenkins add class="empty-list" attribute if there are no templates
if templates.attrib.get('class') == 'empty-list':
templates.attrib.pop('class')
def plugin_module(plugin):
"""
>>> plugin_module('cgcloud.core') # doctest: +ELLIPSIS
<module 'cgcloud.core' from '...'>
>>> plugin_module('cgcloud.foobar')
Traceback (most recent call last):
...
UserError: Cannot find plugin module 'cgcloud.foobar'. Running 'pip install cgcloud-foobar' may fix this.
"""
try:
return import_module(plugin)
except ImportError:
raise UserError(
"Cannot find plugin module '%s'. Running 'pip install %s' may fix this."
% (plugin, plugin.replace('.', '-')))
def resolve_me(self, s, drop_hostname=True):
placeholder = self.current_user_placeholder
if placeholder in s:
me = os.environ.get('CGCLOUD_ME', self.iam_user_name)
if not me:
raise UserError(
"Can't determine current IAM user name. Be sure to put valid AWS credentials "
"in ~/.boto or ~/.aws/credentials. For details, refer to %s. On an EC2 "
"instance that is authorized via IAM roles, you can set the CGCLOUD_ME "
"environment variable (uncommon)." %
'http://boto.readthedocs.org/en/latest/boto_config_tut.html'
)
if drop_hostname:
me = self.drop_hostname(me)
return s.replace(placeholder, me)
else:
return s
def instance_options(self, options, box):
"""
Return dict with keyword arguments to be passed box.prepare()
"""
role_options = box.get_role_options()
supported_options = set(option.name for option in role_options)
actual_options = set(name for name, value in options.role_options)
for name in actual_options - supported_options:
raise UserError("Options %s not supported by role '%s'." %
(name, box.role()))
resolve_me = functools.partial(box.ctx.resolve_me, drop_hostname=False)
return dict(options.role_options,
ec2_keypair_globs=map(resolve_me,
options.ec2_keypair_names),
instance_type=options.instance_type,
virtualization_type=options.virtualization_type,
spot_bid=options.spot_bid,
launch_group=options.launch_group)
def register_ssh_pubkey(self, ec2_keypair_name, ssh_pubkey, force=False):
"""
Import the given OpenSSH public key as a 'key pair' into EC2.
There is no way to get to the actual public key once it has been imported to EC2.
Openstack lets you do that and I don't see why Amazon decided to omit this functionality.
To work around this, we store the public key in S3, identified by the public key's
fingerprint. As long as we always check the fingerprint of the downloaded public SSH key
against that of the EC2 keypair key, this method is resilient against malicious
modifications of the keys stored in S3.
:param ec2_keypair_name: the desired name of the EC2 key pair
:param ssh_pubkey: the SSH public key in OpenSSH's native format, i.e. format that is used in ~/
.ssh/authorized_keys
:param force: overwrite existing EC2 keypair of the given name
"""
fingerprint = ec2_keypair_fingerprint(ssh_pubkey,
reject_private_keys=True)
ec2_keypair = self.ec2.get_key_pair(ec2_keypair_name)
if ec2_keypair is not None:
if ec2_keypair.name != ec2_keypair_name:
raise AssertionError("Key pair names don't match.")
if ec2_keypair.fingerprint != fingerprint:
if force:
self.ec2.delete_key_pair(ec2_keypair_name)
ec2_keypair = None
else:
raise UserError(
"Key pair %s already exists in EC2, but its fingerprint %s is "
"different from the fingerprint %s of the key to be imported. Use "
"the force option to overwrite the existing key pair."
% (ec2_keypair.name, ec2_keypair.fingerprint,
fingerprint))
if ec2_keypair is None:
ec2_keypair = self.ec2.import_key_pair(ec2_keypair_name,
ssh_pubkey)
assert ec2_keypair.fingerprint == fingerprint
self.upload_ssh_pubkey(ssh_pubkey, fingerprint)
self.__publish_key_update_agent_message()
return ec2_keypair
def run(self, options):
zone = options.availability_zone
namespace = options.namespace
ctx = None
try:
ctx = Context(availability_zone=zone, namespace=namespace)
except ValueError as e:
raise UserError(cause=e)
except:
# print the namespace without __me__ substituted
log.error("An error occurred. Using zone '%s' and namespace '%s'",
zone, namespace)
raise
else:
# print the namespace with __me__ substituted
log.info("Using zone '%s' and namespace '%s'",
ctx.availability_zone, ctx.namespace)
return self.run_in_ctx(options, ctx)
finally:
if ctx is not None: ctx.close()
def run_in_ctx(self, options, ctx):
role = self.application.roles.get(options.role)
if role is None: raise UserError("No such role: '%s'" % options.role)
return self.run_on_role(options, ctx, role)
def run_in_ctx(self, options, ctx):
try:
cluster_type = self.application.cluster_types[options.cluster_type]
except KeyError:
raise UserError("Unknown cluster type '%s'" % options.cluster_type)
self.run_on_cluster_type(ctx, options, cluster_type)
def parse_sdists(cls, s):
try:
return [cls.sdist_re.match(sdist).groups() for sdist in s.split()]
except:
raise UserError(
"'%s' is not a valid value for the toil_sdists option." % s)