def checkPrices(ctype=None, cluster_tag=None, max_times=10):
"""
Checks the spot prices
no args compares currently running clusters to possible spot prices
ctype checks for a certain type of image i.e. 'm1.xlarge' and returns
prices in order of cheapest to most expensive
cluster_tag checks only for one running cluster
"""
assert ctype is None or cluster_tag is None, ("you cannot specify a "
"cluster and an image type.")
if ctype == None:
cfg = StarClusterConfig().load()
cm = ClusterManager(cfg)
for cluster in cm.get_clusters():
if cluster_tag is not None and cluster.cluster_tag != cluster_tag:
continue
region, template = readLog(tag=cluster.cluster_tag)
t = "%s - of type(%s) in zone(%s) with bid(%s)" % (
cluster.cluster_tag, cluster.master_instance_type,
cluster._nodes[0].placement, cluster.spot_bid)
print "=" * len(t)
print t
sp = getSpotPrices(cluster.master_instance_type,
max_times=max_times)
print sp[cluster._nodes[0].placement]['print']
print "=" * len(t)
print
print
print "Better/equal prices @"
curr = sp[cluster._nodes[0].placement]['current']
for k, v in sp.iteritems():
if curr >= v['current'] and k != cluster._nodes[0].placement:
print v['print']
else:
sp = getSpotPrices(ctype, max_times=max_times)
a = [(p['current'], k) for k, p in sp.iteritems()]
a.sort()
print "type(%s) from cheapest to most expensive" % ctype
for _, k in a:
print sp[k]['print']
def checkPrices(ctype=None, cluster_tag=None, max_times=10):
"""
Checks the spot prices
no args compares currently running clusters to possible spot prices
ctype checks for a certain type of image i.e. 'm1.xlarge' and returns
prices in order of cheapest to most expensive
cluster_tag checks only for one running cluster
"""
assert ctype is None or cluster_tag is None, ("you cannot specify a "
"cluster and an image type.")
if ctype==None:
cfg = StarClusterConfig().load()
cm = ClusterManager(cfg)
for cluster in cm.get_clusters():
if cluster_tag is not None and cluster.cluster_tag != cluster_tag:
continue
region, template = readLog(tag = cluster.cluster_tag)
t = "%s - of type(%s) in zone(%s) with bid(%s)"% (
cluster.cluster_tag, cluster.master_instance_type,
cluster._nodes[0].placement,cluster.spot_bid)
print "="*len(t)
print t
sp = getSpotPrices( cluster.master_instance_type,
max_times=max_times )
print sp[cluster._nodes[0].placement]['print']
print "="*len(t)
print
print
print "Better/equal prices @"
curr = sp[cluster._nodes[0].placement]['current']
for k,v in sp.iteritems():
if curr >= v['current'] and k != cluster._nodes[0].placement:
print v['print']
else:
sp = getSpotPrices( ctype, max_times=max_times )
a = [(p['current'],k) for k,p in sp.iteritems()]
a.sort()
print "type(%s) from cheapest to most expensive"%ctype
for _,k in a:
print sp[k]['print']
def cloneCluster(new_zone, cluster_tag=None, template=None):
assert cluster_tag is not None or template is not None, "You must provide a cluster template or a cluster tag"
#get full config
cfg = StarClusterConfig().load()
cm = ClusterManager(cfg)
dest_region = new_zone[:-1]
if cluster_tag is not None:
for cluster in cm.get_clusters():
if cluster.cluster_tag == cluster_tag:
region, template = readLog(tag = cluster.cluster_tag)
break
cluster_sect, volume_sect = getClusterConfig(template, cfg)
#get autogenerated portion of config
auto_config = ConfigParser.RawConfigParser()
auto_config.read(os.path.expanduser('~/.starcluster/cfgs/auto-gen.cfg'))
new_cluster_sect = []
new_volume_sect = []
for key,value in cluster_sect:
if key == 'node_image_id' or key=='master_image_id':
source_region = regionFromAMI(value )
if source_region is None:
raise Exception("Cannot find %s in any region." % value)
new_ami = cpAMI( dest_region=dest_region, source_region=source_region, source_image_id=value)
print "NA: ", new_ami
new_cluster_sect.append((key,new_ami))
elif key == 'extends':
if value[:4] != 'base':
print "*"*50
print "ALERT,%s extends non base cluster, CHECK CONFIG"%template
print "*"*50
new_cluster_sect.append((key,'base-%s'%dest_region))
elif key == 'volumes':
value = removeRegion(value)
volume_sec_name = 'volume %s-%s' % (value, new_zone)
counter = 0
while auto_config.has_section(volume_sec_name):
volume_sec_name = 'volume %s-%s-%02d' % (value, new_zone,counter)
counter += 1
for k,v in volume_sect:
if k == 'volume_id':
region_from, zone_from = regionFromVolume( v )
if region_from is None:
raise Exception("Cannot find %s in any region." % v)
new_vol_id = cpVolume(v, new_zone[:-1], new_zone, region_from=region_from)
new_volume_sect.append((k, new_vol_id))
else:
new_volume_sect.append((k,v))
new_cluster_sect.append((key, volume_sec_name.split(' ')[1]))
else:
new_cluster_sect.append((key,value))
cluster_sec_name = 'cluster %s-%s' % (template, new_zone)
counter = 0
while auto_config.has_section(cluster_sec_name):
cluster_sec_name = 'cluster %s-%s-%02d' % (template, new_zone, counter)
counter += 1
auto_config.add_section(cluster_sec_name)
auto_config.add_section(volume_sec_name)
for k,v in new_cluster_sect:
auto_config.set(cluster_sec_name, k,v)
for k,v in new_volume_sect:
auto_config.set(volume_sec_name, k,v)
with open(os.path.expanduser('~/.starcluster/cfgs/auto-gen.cfg'), 'wb') as configfile:
auto_config.write(configfile)
print "starcluster -r %s start --force-spot-master -b <my-bid> -c %s my-cluster"%(new_zone[:-1], cluster_sec_name.split(' ')[1])
def cloneCluster(new_zone, cluster_tag=None, template=None):
assert cluster_tag is not None or template is not None, \
"You must provide a cluster template or a cluster tag"
#get full config
cfg = StarClusterConfig().load()
cm = ClusterManager(cfg)
if new_zone[-1] not in ['a', 'b', 'c', 'd']:
raise Exception("%s does not appear to be a ZONE" % new_zone)
dest_region = new_zone[:-1]
if cluster_tag is not None:
lookup = get_config(cluster_tag=cluster_tag)
if lookup:
template = lookup[0]['template_name']
cluster_cfg = load_config(lookup[0]['config_path'])
else:
raise Exception("Unable to find cluster tag[%s]" % cluster_tag)
else:
lookup = get_config(template_name=template)
if lookup:
template = lookup[0]['template_name']
cluster_cfg = load_config(lookup[0]['config_path'])
else:
raise Exception("Unable to find template[%s]" % cluster_tag)
#get autogenerated portion of config
auto_config = ConfigParser.RawConfigParser()
auto_config.read(
os.path.join(static.STARCLUSTER_CFG_DIR, 'cfgs/auto-gen.cfg'))
new_cluster_sect = []
new_cluster_sect.append(('KEYNAME', static.KEYNAMES[dest_region]))
new_cluster_sect.append(('EXTENDS', template))
new_volume_sect = []
nit_v = ''
if 'node_instance_types' in cluster_cfg:
for nit in cluster_cfg['node_instance_types']:
source_region = regionFromAMI(nit['image'])
new_ami = cpAMI(dest_region=dest_region,
source_region=source_region,
source_image_id=nit['image'])
if len(nit_v) > 0:
nit_v += ', '
if 'type' in nit:
nit_v += ':'.join([nit['type'], new_ami, str(nit["size"])])
else:
nit_v += new_ami
if 'node_instance_type' in cluster_cfg \
and cluster_cfg['node_instance_type']:
if len(nit_v) > 0:
nit_v += ', '
nit_v += cluster_cfg['node_instance_type']
new_cluster_sect.append(('NODE_INSTANCE_TYPE', nit_v))
print cluster_cfg
if 'master_instance_type' in cluster_cfg \
and cluster_cfg['master_instance_type']:
new_cluster_sect.append(
('MASTER_INSTANCE_TYPE', cluster_cfg['master_instance_type']))
if 'master_image_id' in cluster_cfg and cluster_cfg['master_image_id']:
source_region = regionFromAMI(cluster_cfg['master_image_id'])
new_ami = cpAMI(dest_region=dest_region,
source_region=source_region,
source_image_id=cluster_cfg['master_image_id'])
new_cluster_sect.append(('MASTER_IMAGE_ID', new_ami))
if 'node_image_id' in cluster_cfg and cluster_cfg['node_image_id']:
source_region = regionFromAMI(cluster_cfg['node_image_id'])
new_ami = cpAMI(dest_region=dest_region,
source_region=source_region,
source_image_id=cluster_cfg['node_image_id'])
new_cluster_sect.append(('NODE_IMAGE_ID', new_ami))
volumes_names = []
if 'volumes' in cluster_cfg:
for vname, vdesc in cluster_cfg['volumes'].iteritems():
new_volume_sect = []
vname = removeRegion(vname)
volume_sec_name = 'volume %s-%s' % (vname, new_zone)
counter = 0
while auto_config.has_section(volume_sec_name):
volume_sec_name = 'volume %s-%s-%02d' % (vname, new_zone,
counter)
counter += 1
region_from, zone_from = regionFromVolume(vdesc['volume_id'])
if region_from is None:
raise Exception("Cannot find %s in any region." % v)
new_vol_id = cpVolume(vdesc['volume_id'],
new_zone[:-1],
new_zone,
region_from=region_from)
new_volume_sect.append(('VOLUME_ID', new_vol_id))
new_volume_sect.append(('MOUNT_PATH', vdesc['mount_path']))
auto_config.add_section(volume_sec_name)
for k, v in new_volume_sect:
auto_config.set(volume_sec_name, k, v)
volumes_names.append(volume_sec_name.split(' ')[1])
if len(volumes_names) > 0:
new_cluster_sect.append(('VOLUMES', ','.join(volumes_names)))
counter = 0
template = removeRegion(template)
cluster_sec_name = 'cluster %s-%s' % (template, new_zone)
while auto_config.has_section(cluster_sec_name):
cluster_sec_name = 'cluster %s-%s-%02d' % (template, new_zone, counter)
counter += 1
auto_config.add_section(cluster_sec_name)
for k, v in new_cluster_sect:
auto_config.set(cluster_sec_name, k, v)
with open(os.path.join(static.STARCLUSTER_CFG_DIR, 'cfgs/auto-gen.cfg'),
'wb') as configfile:
auto_config.write(configfile)
log.info(("starcluster -r %s start --force-spot-master -b "
"<my-bid> -c %s my-cluster") %
(new_zone[:-1], cluster_sec_name.split(' ')[1]))