def create_zk( ec2_conn, ipa_client, master, subnet_id=None, ip_address=None, instance_type=None): """ Create new Zookeeper """ sts_conn = awscontext.GLOBAL.sts ipa_domain = awscontext.GLOBAL.ipa_domain admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(context.GLOBAL.cell) data = cell['data'] image_id = data['image'] if not image_id.startswith('ami-'): account = sts_conn.get_caller_identity().get('Account') image_id = ec2client.get_image( ec2_conn, owners=[account], name=image_id )['ImageId'] # FIXME; subnet not unique among ZK, not AZ aware if not subnet_id: subnet_id = random.choice(data['subnets']) shortname = master['hostname'].replace('.' + context.GLOBAL.dns_domain, '') if not instance_type: instance_type = 'm5.large' # Instance vars instance_vars = { 'treadmill_cell': context.GLOBAL.cell, 'treadmill_ldap': ','.join(context.GLOBAL.ldap.url), 'treadmill_ldap_suffix': context.GLOBAL.ldap_suffix, 'treadmill_dns_domain': context.GLOBAL.dns_domain, 'treadmill_isa': 'zookeeper', 'treadmill_profile': 'aws', 'treadmill_krb_realm': krb5.get_host_realm(sysinfo.hostname())[0], 'treadmill_zookeeper_myid': str(master['idx']), } # Build user-data and start new instance create_host(ec2_conn=ec2_conn, ipa_client=ipa_client, image_id=image_id, count=1, domain=ipa_domain, secgroup_ids=data['secgroup'], instance_type=instance_type, subnets=[subnet_id], disk=30, instance_vars=instance_vars, role='zookeeper', hostgroups=['zookeepers'], hostname=shortname, ip_address=ip_address) return master['hostname']
def zookeeper(ctx, run, master_id, data_dir): """Installs Treadmill master.""" ctx.obj['PARAMS']['zookeeper'] = context.GLOBAL.zk.url ctx.obj['PARAMS']['ldap'] = context.GLOBAL.ldap.url ctx.obj['PARAMS']['master_id'] = master_id realms = krb5.get_host_realm(sysinfo.hostname()) if realms: ctx.obj['PARAMS']['krb_realm'] = realms[0] if data_dir: ctx.obj['PARAMS']['data_dir'] = data_dir dst_dir = ctx.obj['PARAMS']['dir'] profile = ctx.obj['PARAMS'].get('profile') for master in ctx.obj['PARAMS']['masters']: # pylint: disable=E1136 if int(master['idx']) == int(master_id): ctx.obj['PARAMS'].update({'me': master}) run_sh = None if run: run_sh = os.path.join(dst_dir, 'treadmill', 'bin', 'run.sh') bootstrap.install( 'zookeeper', dst_dir, ctx.obj['PARAMS'], run=run_sh, profile=profile, )
def _configure_locker(tkt_spool_dir, scandir, cell, celluser): """Configure ticket forwarding service.""" if os.path.exists(os.path.join(scandir, cell)): return _LOGGER.info('Configuring ticket locker: %s/%s', scandir, cell) name = cell realms = krb5.get_host_realm(sysinfo.hostname()) krb5ccname = 'FILE:{tkt_spool_dir}/{celluser}@{realm}'.format( tkt_spool_dir=tkt_spool_dir, celluser=celluser, realm=realms[0], ) supervisor.create_service( scandir, name=name, app_run_script=( '{treadmill}/bin/treadmill sproc ' 'tickets locker --tkt-spool-dir {tkt_spool_dir}'.format( treadmill=subproc.resolve('treadmill'), tkt_spool_dir=tkt_spool_dir)), userid='root', environ_dir=os.path.join(scandir, name, 'env'), environ={ 'KRB5CCNAME': krb5ccname, 'TREADMILL_CELL': cell, }, downed=False, trace=None, monitor_policy=None)
def make_host_acl(self, host, perm): """Create host acl in zookeeper. """ realms = krb5.get_host_realm(host) if not realms: _LOGGER.critical('Host %s does not belong to krb realm.', host) raise Exception('Host does not belong to krb5 realm.') host_principal = 'host/{host}@{realm}'.format(host=host, realm=realms[0]) return kazoo.security.make_acl(scheme='sasl', credential=host_principal, read='r' in perm, write='w' in perm, create='c' in perm, delete='d' in perm, admin='a' in perm)
def __init__(self, cors=None, krb_realm=None): self.cell = context.GLOBAL.cell admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(self.cell) self.proid = cell['username'] self.data = cell.get('data') # Default cors origin to top level dns domain. The value is passed to # manifest verbatim, so need to shell escape it. if not cors: last_two = context.GLOBAL.dns_domain.split('.')[-2:] self.cors = '\\.'.join(last_two) else: self.cors = '\\.'.join(cors.strip('.').split('.')) self.krb_realm = krb_realm if not self.krb_realm: realms = krb5.get_host_realm(sysinfo.hostname()) if realms: self.krb_realm = realms[0]
def create_n_servers(count, partition=None, min_on_demand=None, max_on_demand=None, pool=None): """Create new servers in the cell.""" partition = partition or '_default' # FIXME: Import name from treadmill. _LOGGER.info( 'Creating %s servers in %s partition, min on-demand: %s, max: %s', count, partition, min_on_demand, max_on_demand) ec2_conn = awscontext.GLOBAL.ec2 ipa_domain = awscontext.GLOBAL.ipa_domain admin_cell = context.GLOBAL.admin.cell() cell = context.GLOBAL.cell cell_data = admin_cell.get(cell)['data'] admin_part = context.GLOBAL.admin.partition() try: _ldap_data = admin_part.get([partition, cell], dirty=True) partition_data = _ldap_data.get('data', {}) except admin_exceptions.NoSuchObjectResult: partition_data = {} image_id = partition_data.get('image', cell_data['image']) if not image_id.startswith('ami-'): image_id = ec2client.get_image(ec2_conn, owners=['self'], name=image_id)['ImageId'] instance_type = partition_data.get('size', cell_data['size']) instance_types = partition_data.get('instance_types', [instance_type]) spot_instance_types = partition_data.get('spot_instance_types', instance_types) subnets = partition_data.get('subnets', cell_data['subnets']) secgroup_id = partition_data.get('secgroup', cell_data['secgroup']) hostgroups = partition_data.get('hostgroups', cell_data['hostgroups']) instance_profile = partition_data.get('instance_profile', cell_data['instance_profile']) disk_size = int(partition_data.get('disk_size', cell_data['disk_size'])) nshostlocation = cell_data['aws_account'] instance_vars = { 'treadmill_cell': cell, 'treadmill_ldap': ','.join(context.GLOBAL.ldap.url), 'treadmill_ldap_suffix': context.GLOBAL.ldap_suffix, 'treadmill_dns_domain': context.GLOBAL.dns_domain, 'treadmill_isa': 'node', 'treadmill_profile': 'aws', 'treadmill_krb_realm': krb5.get_host_realm(sysinfo.hostname())[0], } tags = [ { 'Key': 'Cell', 'Value': cell }, { 'Key': 'Partition', 'Value': partition }, ] hostnames = _generate_hostnames(ipa_domain, cell, partition, count, min_on_demand=min_on_demand, max_on_demand=max_on_demand) instance_types = _instance_types(instance_types, spot_instance_types) host_params = dict(image_id=image_id, count=1, disk=disk_size, domain=ipa_domain, key=None, secgroup_ids=secgroup_id, role='node', instance_vars=instance_vars, instance_profile=instance_profile, hostgroups=hostgroups, ip_address=None, eni=None, tags=tags, nshostlocation=nshostlocation) if pool: func = functools.partial(_create_hosts_no_exc, instance_types=instance_types, subnets=subnets, cell=cell, partition=partition, **host_params) hosts_created = [] for res, err in pool.map(func, _split_list(hostnames, pool.workers)): if err: raise err hosts_created.extend(res) return hosts_created else: return _create_hosts(hostnames, instance_types, subnets, cell, partition, **host_params)
def ipa525server(port, client_realm, server_realm, authz): """Run IPA525 credential daemon.""" if not server_realm: server_realm = krb5.get_host_realm(socket.getfqdn())[0] ipa525.run_server(port, client_realm, server_realm, authz)
def create_n_servers(count, partition=None): """Create new servers in the cell.""" ipa_client = awscontext.GLOBAL.ipaclient ec2_conn = awscontext.GLOBAL.ec2 sts_conn = awscontext.GLOBAL.sts ipa_domain = awscontext.GLOBAL.ipa_domain admin_srv = admin.Server(context.GLOBAL.ldap.conn) admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(context.GLOBAL.cell) data = cell['data'] image_id = data['image'] if not image_id.startswith('ami-'): account = sts_conn.get_caller_identity().get('Account') image_id = ec2client.get_image(ec2_conn, owners=[account], name=image_id)['ImageId'] instance_type = data['size'] subnets = data['subnets'] secgroup_id = data['secgroup'] hostgroups = data['hostgroups'] instance_profile = data['instance_profile'] disk_size = int(data['disk_size']) hostname_template = '{}-{}-{}'.format(context.GLOBAL.cell, partition if partition else 'node', '{time}') instance_vars = { 'treadmill_cell': context.GLOBAL.cell, 'treadmill_ldap': ','.join(context.GLOBAL.ldap.url), 'treadmill_ldap_suffix': context.GLOBAL.ldap_suffix, 'treadmill_dns_domain': context.GLOBAL.dns_domain, 'treadmill_isa': 'node', 'treadmill_profile': 'aws', 'treadmill_krb_realm': krb5.get_host_realm(sysinfo.hostname())[0], } # FIXME: Add Partition: $partition to tags when Autoscaler is cell aware tags = [{'Key': 'Cell', 'Value': context.GLOBAL.cell}] key = None for idx in range(0, count): hostnames = hostmanager.create_host(ipa_client=ipa_client, ec2_conn=ec2_conn, image_id=image_id, count=1, disk=disk_size, domain=ipa_domain, key=key, secgroup_ids=secgroup_id, instance_type=instance_type, subnets=subnets, role='node', instance_vars=instance_vars, instance_profile=instance_profile, hostgroups=hostgroups, hostname=hostname_template, ip_address=None, eni=None, tags=tags) # Count is one, but it is more robust to treat it as list. for hostname in hostnames: print(hostname) attrs = {'cell': context.GLOBAL.cell, 'partition': partition} admin_srv.create(hostname, attrs)
def info_cmd(tkt_spool_dir, tkt_info_dir, tkt_realms): """Monitor tickets and dump ticket info as json file.""" realms = krb5.get_host_realm(sysinfo.hostname()) if tkt_realms: realms.extend(tkt_realms) def _is_valid(princ): if princ.startswith('.'): return False for realm in realms: if princ.endswith(realm): return True return False def handle_tkt_event(tkt_file): """Handle ticket created event. """ princ = os.path.basename(tkt_file) if not _is_valid(princ): return infofile = os.path.join(tkt_info_dir, princ) _LOGGER.info('Processing: %s', princ) try: os.environ['KRB5CCNAME'] = os.path.join(tkt_spool_dir, princ) creds = gssapi.creds.Credentials(usage='initiate') with open(infofile, 'wb') as f: f.write( json.dumps({ 'expires_at': int(time.time()) + creds.lifetime }).encode()) except gssapi.raw.GSSError as gss_err: fs.rm_safe(infofile) finally: del os.environ['KRB5CCNAME'] def handle_tkt_delete(tkt_file): """Delete ticket info. """ princ = os.path.basename(tkt_file) if not _is_valid(princ): return infofile = os.path.join(tkt_info_dir, princ) _LOGGER.info('Deleting: %s', princ) fs.rm_safe(infofile) watch = dirwatch.DirWatcher(tkt_spool_dir) watch.on_created = handle_tkt_event watch.on_deleted = handle_tkt_delete for tkt_file in os.listdir(tkt_spool_dir): handle_tkt_event(tkt_file) while True: if watch.wait_for_events(timeout=60): watch.process_events(max_events=100)