def __init__(self, profile, ec2con=None, azstrat=None, sleep_time=5):
self.profile = profile
self.sleep_time = sleep_time
if not ec2con and profile.region:
self.ec2con = ec2.connect_to_region(profile.region)
else:
self.ec2con = ec2con
if not azstrat:
self.azstrat = TagBasedStrategy(ec2con=self.ec2con, tags=profile.discovery_tags)
else:
self.azstrat = azstrat
class GenericLauncher(object):
"""Takes care of launching a new ec2 nodes.
Requires a warmachine.config.Profile which contains items like the tags.
"""
def __init__(self, profile, ec2con=None, azstrat=None, sleep_time=5):
self.profile = profile
self.sleep_time = sleep_time
if not ec2con and profile.region:
self.ec2con = ec2.connect_to_region(profile.region)
else:
self.ec2con = ec2con
if not azstrat:
self.azstrat = TagBasedStrategy(ec2con=self.ec2con, tags=profile.discovery_tags)
else:
self.azstrat = azstrat
def launch_and_configure(self):
"""Launch one or more new nodes for given cluster
given the configured profile.
"""
# make sure any security groups required are created if not already
for group in self.profile.security_group_defs:
try:
group.create(self.ec2con)
except SecurityGroupExistsError:
logger.debug("Skipping sec group '%s' already exists", group.name)
instances = self.launch_nodes()
time.sleep(self.sleep_time)
for inst in instances:
logger.info('Begin configuration of instance %s', inst)
while True:
try:
inst.update()
host = inst.public_dns_name
self.verify_and_configure(inst, host=host)
break
except socket.error as serr:
if serr.errno not in [errno.EHOSTUNREACH, errno.ECONNREFUSED, errno.ETIMEDOUT]:
raise serr
# sleep if not ready yet
logger.info('ssh not ready on instance %s, going to sleep %s seconds', inst.id, self.sleep_time)
time.sleep(self.sleep_time)
def launch_nodes(self):
"""Launch a single node per the equiped profile
"""
p = self.profile
# now lets launch some instances
badzones = []
attempts = 0
while True:
attempts = attempts + 1
if attempts > 5:
raise Exception('After five attempts could not create instance(s)')
zone = self.azstrat.get_zone(ignores=badzones)
logger.info("Going to attempt to bring up instance in " \
"region '%s' zone '%s" % (p.region, zone))
try:
reservation = self.ec2con.run_instances(
image_id=p.image_id,
min_count=1,
max_count=1,
key_name=p.key_name,
security_groups=[sg.name for sg in p.security_group_defs],
instance_type=p.instance_type,
placement=zone,
ebs_optimized=p.ebs_optimized,)
except EC2ResponseError as ec2re:
# if it's a AZ going nutty, we can try others
if 'The requested Availability Zone is currently constrained' in ec2re.body:
badzones.append(zone)
logger.info("It appears that AZ '%s' is currently " \
"constrained, going to use other AZs for now" % zone)
elif 'Please retry your request by not specifying an Availability Zone or choosing' in ec2re.body:
badzones.append(zone)
logger.info('Zone %s does not do %s instances' % (zone, p.instance_type))
else:
raise ec2re
else:
break
instances = reservation.instances
# do tagging of instances
for instance in instances:
while True:
try:
tags = p.build_instance_tags(instanceid=instance.id)
for name, value in tags.items():
instance.add_tag(name, value)
break
except EC2ResponseError as ec2re:
time.sleep(1)
continue
def get_statuses(instances):
s = set()
for instance in instances:
status = instance.update()
s.add(instance.state)
return s
statuses = get_statuses(instances)
while 'pending' in statuses:
time.sleep(p.sleep_time)
statuses = get_statuses(instances)
logger.info('Launched instances: %s' % instances)
return instances
@wssh.must_have_paramiko_client
def verify_and_configure(self, inst):
if hasattr(self.profile, 'encrypted_volume_defs'):
voldefs = self.profile.encrypted_volume_defs
for voldef in voldefs:
self.decrypt_and_mount_volume(voldef)
self.configure_hostname(inst)
self.configure_node(inst)
@wssh.must_have_paramiko_client
def wait_for_device(self, device):
pclient, p, log = self.pclient, self.profile, logger.info
host = self.host
log("connected to host '%s', waiting for ebs volumes to be booted" % (host))
# check the ebs volume is there, it sometimes takes a moment
_, stdout, _ = pclient.exec_command("[ -e %s ] && echo OK" % (device))
val = stdout.read().strip()
while val != 'OK':
log("waiting on ebs device: '%s'" % device)
time.sleep(self.sleep_time)
_, stdout, _ = pclient.exec_command("[ -e %s ] && echo OK" % (device))
val = stdout.read().strip()
@wssh.must_have_paramiko_client
def decrypt_and_mount_volume(self, volume_def):
v = volume_def
pclient, p, log = self.pclient, self.profile, logger.info
self.wait_for_device(v['dev'])
# open the encypted volume
stdin, stdout, stderr = pclient.exec_command(
'sudo cryptsetup luksOpen %s %s' % (v['dev'], v['mapper']))
stdin.write("%s\n" % v['pass'])
stdin.flush()
stdin.channel.shutdown_write()
log("luksOpen executed, output: '%s'" % stdout.read())
# mount the open volume
stdin, stdout, stderr = pclient.exec_command(
'sudo mount /dev/mapper/%s %s' % (v['mapper'], v['mount']))
log(stdout.read())
log(stderr.read())
log("volume %s mounted to %s" % (v['mapper'], v['mount']))
@wssh.must_have_paramiko_client
def configure_node(self, inst):
"""Configure elastic search needed.
"""
logger.info('In configure_node for instance %s, you must override this" \
"method to perform actions after instance launch', inst.id)
@wssh.must_have_paramiko_client
def configure_hostname(self, instance):
pclient, p, log = self.pclient, self.profile, logger.info
hostname = p.build_hostname(instanceid=instance.id)
if not hostname:
log("Skipped hostname configuration, unable to build hostname!")
return
stdin, stdout, stderr = pclient.exec_command(
"sudo su -c 'echo \"127.0.0.1 %s\" >> /etc/hosts'" % (hostname))
log(stdout.read())
log(stderr.read())
stdin, stdout, stderr = pclient.exec_command(
"sudo su -c 'echo \"%s\" > /etc/hostname'" % (hostname))
log(stdout.read())
log(stderr.read())
stdin, stdout, stderr = pclient.exec_command(
"sudo hostname %s" % (hostname))
log(stdout.read())
log(stderr.read())
