def handle_completion(self):
if self.is_completion_active():
gparser = self.create_global_parser(no_usage=True, add_help=False)
# set sys.path to COMP_LINE if it exists
self._init_completion()
# fetch the global options
gopts = self.get_global_opts()
# try to load StarClusterConfig into global options
if gopts:
try:
cfg = config.StarClusterConfig(gopts.CONFIG)
cfg.load()
except exception.ConfigError:
cfg = None
gopts.CONFIG = cfg
scmap = {}
for sc in commands.all_cmds:
sc.gopts = gopts
for n in sc.names:
scmap[n] = sc
listcter = completion.ListCompleter(scmap.keys())
subcter = completion.NoneCompleter()
completion.autocomplete(gparser,
listcter,
None,
subcter,
subcommands=scmap)
sys.exit(1)
def parse_subcommands(self, gparser=None):
"""
Parse global arguments, find subcommand from list of subcommand
objects, parse local subcommand arguments and return a tuple of
global options, selected command object, command options, and
command arguments.
Call execute() on the command object to run. The command object has
members 'gopts' and 'opts' set for global and command options
respectively, you don't need to call execute with those but you could
if you wanted to.
"""
gparser = gparser or self.gparser
# parse global options.
gopts, args = gparser.parse_args()
if not args:
gparser.print_help()
raise SystemExit("\nError: you must specify an action.")
# set debug level if specified
if gopts.DEBUG:
console.setLevel(logger.DEBUG)
config.DEBUG_CONFIG = True
# load StarClusterConfig into global options
try:
cfg = config.StarClusterConfig(gopts.CONFIG)
cfg.load()
except exception.ConfigNotFound, e:
log.error(e.msg)
e.display_options()
sys.exit(1)
def test_config_dne(self):
tmp_file = tempfile.NamedTemporaryFile()
non_existent_file = tmp_file.name
tmp_file.close()
assert not os.path.exists(non_existent_file)
try:
config.StarClusterConfig(non_existent_file, cache=True).load()
except exception.ConfigNotFound:
pass
else:
raise Exception('config loaded non-existent config file %s' %
non_existent_file)
def test_empty_config(self):
"""
Test that reading an empty config generates no errors and that aws
credentials can be read from the environment.
"""
aws_key = 'testkey'
aws_secret_key = 'testsecret'
os.environ['AWS_ACCESS_KEY_ID'] = aws_key
os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_key
tmp_file = tempfile.NamedTemporaryFile()
cfg = config.StarClusterConfig(tmp_file.name, cache=True).load()
assert cfg.aws['aws_access_key_id'] == aws_key
assert cfg.aws['aws_secret_access_key'] == aws_secret_key
def test_import_keypair_fingerprint(keypair):
"""
TODO: move this to 'live' tests
"""
from starcluster import config
cfg = config.StarClusterConfig().load()
ec2 = cfg.get_easy_ec2()
key_location = cfg.get_key(keypair).key_location
localfprint = get_public_rsa_fingerprint(key_location)
ec2fprint = ec2.get_keypair(keypair).fingerprint
print 'local fingerprint: %s' % localfprint
print ' ec2 fingerprint: %s' % ec2fprint
assert localfprint == ec2fprint
def test_invalid_config(self):
"""
Test that reading a non-INI formatted file raises an exception
"""
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(
"<html>random garbage file with no section headings</html>")
tmp_file.flush()
try:
config.StarClusterConfig(tmp_file.name, cache=True).load()
except exception.ConfigHasNoSections:
pass
else:
raise Exception("config allows non-INI formatted files")
def test_create_keypair_fingerprint(keypair=None):
"""
TODO: move this to 'live' tests
"""
from starcluster import config
cfg = config.StarClusterConfig().load()
ec2 = cfg.get_easy_ec2()
if keypair is None:
keypair = cfg.keys.keys()[0]
key_location = cfg.get_key(keypair).key_location
localfprint = get_private_rsa_fingerprint(key_location)
ec2fprint = ec2.get_keypair(keypair).fingerprint
print 'local fingerprint: %s' % localfprint
print ' ec2 fingerprint: %s' % ec2fprint
assert localfprint == ec2fprint
def parse_subcommands(self, gparser, subcmds):
"""
Parse given global arguments, find subcommand from given list of
subcommand objects, parse local arguments and return a tuple of
global options, selected command object, command options, and
command arguments.
Call execute() on the command object to run. The command object has
members 'gopts' and 'opts' set for global and command options
respectively, you don't need to call execute with those but you could
if you wanted to.
"""
print self.get_description()
# Build map of name -> command and docstring.
cmds_header = 'Available Commands:'
gparser.usage += '\n\n%s\n' % cmds_header
gparser.usage += '%s\n' % ('-' * len(cmds_header))
gparser.usage += "NOTE: Pass --help to any command for a list of its "
gparser.usage += 'options and detailed usage information\n\n'
for sc in subcmds:
helptxt = sc.__doc__.splitlines()[3].strip()
gparser.usage += '- %s: %s\n' % (', '.join(sc.names),
helptxt)
for n in sc.names:
assert n not in self.subcmds_map
self.subcmds_map[n] = sc
# Declare and parse global options.
gparser.disable_interspersed_args()
gopts, args = gparser.parse_args()
if not args:
gparser.print_help()
raise SystemExit("\nError: you must specify an action.")
subcmdname, subargs = args[0], args[1:]
# set debug level if specified
if gopts.DEBUG:
console.setLevel(logger.DEBUG)
# load StarClusterConfig into global options
try:
cfg = config.StarClusterConfig(gopts.CONFIG)
cfg.load()
except exception.ConfigNotFound, e:
log.error(e.msg)
e.display_options()
sys.exit(1)
def ec2(self):
if not self._ec2:
cfg = config.StarClusterConfig().load()
self._ec2 = cfg.get_easy_ec2()
return self._ec2
def deploy_img(img_path,
vol_size,
arch,
region,
src_ami,
dev=None,
kernel_id=None,
ramdisk_id=None,
platform=None,
remove_old=False,
**cluster_kwargs):
"""
Deploy a filesystem image as a new AMI in a given region.
This method creates a 1-node host cluster in the desired `region`, copies
the filesystem image to the cluster, creates and attaches a new EBS volume
with size `vol_size`, installs the image onto the new EBS volume, creates a
snapshot of the resulting volume and registers a new AMI in the `region`.
"""
cfg = config.StarClusterConfig().load()
ec2 = cfg.get_easy_ec2()
ec2.connect_to_region(region)
src_img = ec2.get_image(src_ami)
kernel_id = kernel_id or src_img.kernel_id
ramdisk_id = ramdisk_id or src_img.ramdisk_id
itypemap = dict(i386='m1.small', x86_64='m1.large')
dev = dev or dict(i386='/dev/sdj', x86_64='/dev/sdz')[arch]
cm = cluster.ClusterManager(cfg, ec2)
try:
log.info("Checking for existing imghost cluster")
cl = cm.get_cluster('imghost')
log.info("Using existing imghost cluster")
except exception.ClusterDoesNotExist:
log.info("No imghost cluster found, creating...")
default = cm.get_default_cluster_template()
cl = cm.get_cluster_template(default, 'imghost')
keys = ec2.get_keypairs()
key = None
for k in keys:
if cfg.keys.has_key(k.name):
key = cfg.keys.get(k.name)
key['keyname'] = k.name
break
if key:
cluster_kwargs.update(key)
hostitype = itypemap[src_img.architecture]
cluster_kwargs.update(
dict(cluster_size=1,
cluster_shell="bash",
node_image_id=src_ami,
node_instance_type=hostitype))
cl.update(cluster_kwargs)
cl.start(create_only=True, validate=True)
cl.wait_for_cluster()
host = cl.master_node
log.info("Copying %s to /mnt on master..." % img_path)
host.ssh.put(img_path, '/mnt/')
bname = os.path.basename(img_path)
if bname.endswith('.tar.gz'):
log.info("Extracting image(s)...")
host.ssh.execute('cd /mnt && tar xvzf %s' % bname)
bname = bname.replace('.tar.gz', '')
if not host.ssh.isfile('/mnt/%s' % bname):
raise exception.BaseException("/mnt/%s does not exist" % bname)
log.info("Creating EBS volume")
vol = ec2.create_volume(vol_size, host.placement)
log.info("Attaching EBS volume %s to master as %s" % (vol.id, dev))
vol.attach(host.id, dev)
log.info("Waiting for drive to attach...")
s = spinner.Spinner()
s.start()
realdev = '/dev/xvd%s' % dev[-1]
while not host.ssh.path_exists(realdev):
time.sleep(10)
s.stop()
log.info("Installing image on volume %s ..." % vol.id)
host.ssh.execute("cat /mnt/%s > %s" % (bname, realdev))
log.info("Checking filesystem...")
host.ssh.execute("e2fsck -pf %s" % realdev)
log.info("Resizing filesystem to fit EBS volume...")
host.ssh.execute("resize2fs %s" % realdev)
vol.detach()
while vol.update() != 'available':
time.sleep(10)
xarch = arch
if xarch == 'i386':
xarch = 'x86'
snapdesc = 'StarCluster %s %s EBS AMI Snapshot' % (platform, xarch)
snap = ec2.create_snapshot(vol,
description=snapdesc,
wait_for_snapshot=True)
vol.delete()
bmap = ec2.create_root_block_device_map(snap.id, add_ephemeral_drives=True)
imgname = string.lower(platform.replace(' ', '-'))
imgname = 'starcluster-base-%s-%s' % (imgname, xarch)
imgdesc = 'StarCluster Base %s %s (%s)' % (platform, xarch,
string.capitalize(region))
oldimg = ec2.get_images(filters=dict(name=imgname))
if oldimg:
oldimg = oldimg[0]
oldsnap = ec2.get_snapshot(
oldimg.block_device_mapping['/dev/sda1'].snapshot_id)
if remove_old:
log.info("Deregistering old AMI: %s" % oldimg.id)
oldimg.deregister()
log.info("Deleting old snapshot: %s" % oldsnap.id)
oldsnap.delete()
else:
log.info("Existing image %s already has name '%s'" %
(oldimg.id, imgname))
log.info("Please remove old image %s and snapshot %s" %
(oldimg.id, oldsnap.id))
log.info("Then register new AMI with snapshot %s and name '%s'" %
(snap.id, imgname))
return
img = ec2.register_image(name=imgname,
description=imgdesc,
architecture=arch,
kernel_id=kernel_id,
ramdisk_id=ramdisk_id,
root_device_name='/dev/sda1',
block_device_map=bmap)
return img
def config():
cfg = sconfig.StarClusterConfig().load()
assert cfg.aws.aws_access_key_id
assert cfg.aws.aws_secret_access_key
return cfg
def _prep_for_completion(self):
self._cfg = config.StarClusterConfig().load()
self._ec2 = self._cfg.get_easy_ec2()
def addopts(self, parser):
#### CHANGEDprint Dumper ;
log.info(
"0.92.rc2 XXXXX commands.start.addopts utils.config_file: %s",
utils.config_file)
#cfg = config.StarClusterConfig().load()
cfg = config.StarClusterConfig(utils.config_file).load()
templates = cfg.get_cluster_names().keys()
opt = parser.add_option("-c", "--cluster-template", action="store",
dest="cluster_template", choices=templates,
default=None,
help="cluster template to use " + \
"from the config file")
parser.add_option("-x", "--no-create", dest="no_create",
action="store_true", default=False,
help="Do not launch new EC2 instances when " + \
"starting cluster (use existing instances instead)")
parser.add_option("-o", "--create-only", dest="create_only",
action="store_true", default=False,
help="Only launch/start EC2 instances, " + \
"do not perform any setup routines")
parser.add_option("-v", "--validate-only", dest="validate_only",
action="store_true", default=False,
help="Only validate cluster settings, do " + \
"not start a cluster")
parser.add_option("-V",
"--skip-validation",
dest="validate",
action="store_false",
default=True,
help="Do not validate cluster settings")
parser.add_option("-l",
"--login-master",
dest="login_master",
action="store_true",
default=False,
help="ssh to ec2 cluster master node after launch")
parser.add_option("-q",
"--disable-queue",
dest="disable_queue",
action="store_true",
default=None,
help="Do not configure a queueing system (SGE)")
parser.add_option("-r", "--refresh-interval", dest="refresh_interval",
type="int", action="callback", default=None,
callback=self._positive_int,
help="Refresh interval when waiting for cluster " + \
"nodes to come up (default: 30)")
parser.add_option("-b", "--bid", dest="spot_bid", action="store",
type="float", default=None,
help="Requests spot instances instead of flat " + \
"rate instances. Uses SPOT_BID as max bid for " + \
"the request.")
if optcomplete:
opt.completer = optcomplete.ListCompleter(opt.choices)
parser.add_option("-d", "--description", dest="cluster_description",
action="store", type="string",
default="Cluster requested at %s" % \
time.strftime("%Y%m%d%H%M"),
help="brief description of cluster")
parser.add_option("-s",
"--cluster-size",
dest="cluster_size",
action="callback",
type="int",
default=None,
callback=self._positive_int,
help="number of ec2 instances to launch")
parser.add_option("-u", "--cluster-user", dest="cluster_user",
action="store", type="string", default=None,
help="name of user to create on cluster " + \
"(defaults to sgeadmin)")
opt = parser.add_option("-S", "--cluster-shell", dest="cluster_shell",
action="store",
choices=static.AVAILABLE_SHELLS.keys(),
default=None,
help="shell for cluster user " + \
"(defaults to bash)")
if optcomplete:
opt.completer = optcomplete.ListCompleter(opt.choices)
parser.add_option("-m",
"--master-image-id",
dest="master_image_id",
action="store",
type="string",
default=None,
help="AMI to use when launching master")
parser.add_option("-n",
"--node-image-id",
dest="node_image_id",
action="store",
type="string",
default=None,
help="AMI to use when launching nodes")
parser.add_option("-I",
"--master-instance-type",
dest="master_instance_type",
action="store",
choices=static.INSTANCE_TYPES.keys(),
default=None,
help="specify machine type for the master instance")
opt = parser.add_option("-i", "--node-instance-type",
dest="node_instance_type", action="store",
choices=static.INSTANCE_TYPES.keys(),
default=None,
help="specify machine type for the node " + \
"instances")
if optcomplete:
opt.completer = optcomplete.ListCompleter(opt.choices)
parser.add_option("-a",
"--availability-zone",
dest="availability_zone",
action="store",
type="string",
default=None,
help="availability zone to launch ec2 instances in")
parser.add_option("-k", "--keyname", dest="keyname", action="store",
type="string", default=None,
help="name of the AWS keypair to use when " + \
"launching the cluster")
parser.add_option("-K",
"--key-location",
dest="key_location",
action="store",
type="string",
default=None,
metavar="FILE",
help="path to ssh private key used for this cluster")
from starcluster import config
from starcluster import cluster
cfg = config.StarClusterConfig().load()
ec2 = cfg.get_easy_ec2()
cm = cluster.ClusterManager(cfg, ec2=ec2)
cl = cm.get_cluster("your_running_cluster_tag")
print len(cl.running_nodes)
