def __init__(self, profile, ec2con=None, azstrat=None, sleep_time=5): self.profile = profile self.sleep_time = sleep_time if not ec2con and profile.region: self.ec2con = ec2.connect_to_region(profile.region) else: self.ec2con = ec2con if not azstrat: self.azstrat = TagBasedStrategy(ec2con=self.ec2con, tags=profile.discovery_tags) else: self.azstrat = azstrat
class GenericLauncher(object): """Takes care of launching a new ec2 nodes. Requires a warmachine.config.Profile which contains items like the tags. """ def __init__(self, profile, ec2con=None, azstrat=None, sleep_time=5): self.profile = profile self.sleep_time = sleep_time if not ec2con and profile.region: self.ec2con = ec2.connect_to_region(profile.region) else: self.ec2con = ec2con if not azstrat: self.azstrat = TagBasedStrategy(ec2con=self.ec2con, tags=profile.discovery_tags) else: self.azstrat = azstrat def launch_and_configure(self): """Launch one or more new nodes for given cluster given the configured profile. """ # make sure any security groups required are created if not already for group in self.profile.security_group_defs: try: group.create(self.ec2con) except SecurityGroupExistsError: logger.debug("Skipping sec group '%s' already exists", group.name) instances = self.launch_nodes() time.sleep(self.sleep_time) for inst in instances: logger.info('Begin configuration of instance %s', inst) while True: try: inst.update() host = inst.public_dns_name self.verify_and_configure(inst, host=host) break except socket.error as serr: if serr.errno not in [errno.EHOSTUNREACH, errno.ECONNREFUSED, errno.ETIMEDOUT]: raise serr # sleep if not ready yet logger.info('ssh not ready on instance %s, going to sleep %s seconds', inst.id, self.sleep_time) time.sleep(self.sleep_time) def launch_nodes(self): """Launch a single node per the equiped profile """ p = self.profile # now lets launch some instances badzones = [] attempts = 0 while True: attempts = attempts + 1 if attempts > 5: raise Exception('After five attempts could not create instance(s)') zone = self.azstrat.get_zone(ignores=badzones) logger.info("Going to attempt to bring up instance in " \ "region '%s' zone '%s" % (p.region, zone)) try: reservation = self.ec2con.run_instances( image_id=p.image_id, min_count=1, max_count=1, key_name=p.key_name, security_groups=[sg.name for sg in p.security_group_defs], instance_type=p.instance_type, placement=zone, ebs_optimized=p.ebs_optimized,) except EC2ResponseError as ec2re: # if it's a AZ going nutty, we can try others if 'The requested Availability Zone is currently constrained' in ec2re.body: badzones.append(zone) logger.info("It appears that AZ '%s' is currently " \ "constrained, going to use other AZs for now" % zone) elif 'Please retry your request by not specifying an Availability Zone or choosing' in ec2re.body: badzones.append(zone) logger.info('Zone %s does not do %s instances' % (zone, p.instance_type)) else: raise ec2re else: break instances = reservation.instances # do tagging of instances for instance in instances: while True: try: tags = p.build_instance_tags(instanceid=instance.id) for name, value in tags.items(): instance.add_tag(name, value) break except EC2ResponseError as ec2re: time.sleep(1) continue def get_statuses(instances): s = set() for instance in instances: status = instance.update() s.add(instance.state) return s statuses = get_statuses(instances) while 'pending' in statuses: time.sleep(p.sleep_time) statuses = get_statuses(instances) logger.info('Launched instances: %s' % instances) return instances @wssh.must_have_paramiko_client def verify_and_configure(self, inst): if hasattr(self.profile, 'encrypted_volume_defs'): voldefs = self.profile.encrypted_volume_defs for voldef in voldefs: self.decrypt_and_mount_volume(voldef) self.configure_hostname(inst) self.configure_node(inst) @wssh.must_have_paramiko_client def wait_for_device(self, device): pclient, p, log = self.pclient, self.profile, logger.info host = self.host log("connected to host '%s', waiting for ebs volumes to be booted" % (host)) # check the ebs volume is there, it sometimes takes a moment _, stdout, _ = pclient.exec_command("[ -e %s ] && echo OK" % (device)) val = stdout.read().strip() while val != 'OK': log("waiting on ebs device: '%s'" % device) time.sleep(self.sleep_time) _, stdout, _ = pclient.exec_command("[ -e %s ] && echo OK" % (device)) val = stdout.read().strip() @wssh.must_have_paramiko_client def decrypt_and_mount_volume(self, volume_def): v = volume_def pclient, p, log = self.pclient, self.profile, logger.info self.wait_for_device(v['dev']) # open the encypted volume stdin, stdout, stderr = pclient.exec_command( 'sudo cryptsetup luksOpen %s %s' % (v['dev'], v['mapper'])) stdin.write("%s\n" % v['pass']) stdin.flush() stdin.channel.shutdown_write() log("luksOpen executed, output: '%s'" % stdout.read()) # mount the open volume stdin, stdout, stderr = pclient.exec_command( 'sudo mount /dev/mapper/%s %s' % (v['mapper'], v['mount'])) log(stdout.read()) log(stderr.read()) log("volume %s mounted to %s" % (v['mapper'], v['mount'])) @wssh.must_have_paramiko_client def configure_node(self, inst): """Configure elastic search needed. """ logger.info('In configure_node for instance %s, you must override this" \ "method to perform actions after instance launch', inst.id) @wssh.must_have_paramiko_client def configure_hostname(self, instance): pclient, p, log = self.pclient, self.profile, logger.info hostname = p.build_hostname(instanceid=instance.id) if not hostname: log("Skipped hostname configuration, unable to build hostname!") return stdin, stdout, stderr = pclient.exec_command( "sudo su -c 'echo \"127.0.0.1 %s\" >> /etc/hosts'" % (hostname)) log(stdout.read()) log(stderr.read()) stdin, stdout, stderr = pclient.exec_command( "sudo su -c 'echo \"%s\" > /etc/hostname'" % (hostname)) log(stdout.read()) log(stderr.read()) stdin, stdout, stderr = pclient.exec_command( "sudo hostname %s" % (hostname)) log(stdout.read()) log(stderr.read())