def insert(cell, idx, hostname, client_port, jmx_port, followers_port, election_port, kafka_client_port): """Add master server to a cell""" admin_cell = admin.Cell(context.GLOBAL.ldap.conn) data = { 'idx': int(idx), 'hostname': hostname, 'zk-client-port': client_port, 'zk-jmx-port': jmx_port, 'zk-followers-port': followers_port, 'zk-election-port': election_port, } if kafka_client_port is not None: data['kafka-client-port'] = kafka_client_port attrs = {'masters': [data]} try: admin_cell.update(cell, attrs) cli.out(formatter(admin_cell.get(cell, dirty=True))) except ldap_exceptions.LDAPNoSuchObjectResult: click.echo('Cell does not exist: %s' % cell, err=True)
def configure(cell, version, root, location, username, data, status, traits, zk_auth_scheme, manifest): """Create, get or modify cell configuration""" admin_cell = admin.Cell(context.GLOBAL.ldap.conn) attrs = {} if manifest: with io.open(manifest, 'rb') as fd: attrs = yaml.load(stream=fd) if version: attrs['version'] = version if root: if root == '-': root = None attrs['root'] = root if location: attrs['location'] = location if username: attrs['username'] = username if status: attrs['status'] = status if traits: attrs['traits'] = traits if zk_auth_scheme: attrs['zk-auth-scheme'] = zk_auth_scheme if data: with io.open(data, 'rb') as fd: attrs['data'] = yaml.load(stream=fd) if attrs: try: admin_cell.create(cell, attrs) except ldap_exceptions.LDAPEntryAlreadyExistsResult: admin_cell.update(cell, attrs) try: cli.out(formatter(admin_cell.get(cell, dirty=bool(attrs)))) except ldap_exceptions.LDAPNoSuchObjectResult: click.echo('Cell does not exist: %s' % cell, err=True)
def __init__(self, cors=None, krb_realm=None): self.cell = context.GLOBAL.cell admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(self.cell) self.proid = cell['username'] self.data = cell.get('data') # Default cors origin to top level dns domain. The value is passed to # manifest verbatim, so need to shell escape it. if not cors: last_two = context.GLOBAL.dns_domain.split('.')[-2:] self.cors = '\\.'.join(last_two) else: self.cors = '\\.'.join(cors.strip('.').split('.')) self.krb_realm = krb_realm if not self.krb_realm: realms = krb5.get_host_realm(sysinfo.hostname()) if realms: self.krb_realm = realms[0]
def configure(cell, version, root, location, username, archive_server, archive_username, ssq_namespace, data, manifest): """Create, get or modify cell configuration""" admin_cell = admin.Cell(context.GLOBAL.ldap.conn) attrs = {} if manifest: with open(manifest, 'rb') as fd: attrs = yaml.load(fd.read()) if version: attrs['version'] = version if root: if root == '-': root = None attrs['root'] = root if location: attrs['location'] = location if username: attrs['username'] = username if archive_server: attrs['archive-server'] = archive_server if archive_server: attrs['archive-username'] = archive_username if ssq_namespace: attrs['ssq-namespace'] = ssq_namespace if data: with open(data, 'rb') as fd: attrs['data'] = yaml.load(fd.read()) if attrs: try: admin_cell.create(cell, attrs) except ldap3.LDAPEntryAlreadyExistsResult: admin_cell.update(cell, attrs) try: cli.out(formatter(admin_cell.get(cell))) except ldap3.LDAPNoSuchObjectResult: click.echo('Cell does not exist: %s' % cell, err=True)
def configure_data_cmd(aws_account, aws_admin, aws_region, disk_size, docker_registries, hostgroups, image, instance_profile, realm, secgroup, size, subnets, s3_registry_bucket, tls_ca_cert, tls_host_cert, tls_host_key): """Configure cell data.""" admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(context.GLOBAL.cell) data = cell.get('data', {}) tls_certs = data.get('tls_certs', {}) _set(tls_certs, 'ca_cert', tls_ca_cert) _set(tls_certs, 'host_cert', tls_host_cert) _set(tls_certs, 'host_key', tls_host_key) if not any(tls_certs.values()): tls_certs = '-' modified = _set(data, 'aws_account', aws_account) modified = _set(data, 'aws_admin', aws_admin) or modified modified = _set(data, 'aws_region', aws_region) or modified modified = _set(data, 'docker_registries', docker_registries) or modified modified = _set(data, 'disk_size', disk_size) or modified modified = _set(data, 'hostgroups', hostgroups) or modified modified = _set(data, 'image', image) or modified modified = _set(data, 'instance_profile', instance_profile) or modified modified = _set(data, 'realm', realm) or modified modified = _set(data, 'secgroup', secgroup) or modified modified = _set(data, 'size', size) or modified modified = _set(data, 'subnets', subnets) or modified modified = _set(data, 's3_registry_bucket', s3_registry_bucket) or modified modified = _set(data, 'tls_certs', tls_certs, unset_value={}) or modified if modified: admin_cell.update(context.GLOBAL.cell, {'data': data}) cli.out(formatter(data))
def test_cell_to_entry(self): """Tests conversion of cell to ldap entry.""" cell = { '_id': 'test', 'username': '******', 'location': 'x', 'archive-server': 'y', 'archive-username': '******', 'version': '1.2.3', 'root': '', 'ssq-namespace': 's', 'masters': [ {'idx': 1, 'hostname': 'abc', 'zk-client-port': 5000, 'zk-jmx-port': 6000, 'zk-followers-port': 7000, 'zk-election-port': 8000} ] } cell_admin = admin.Cell(None) self.assertEquals(cell, cell_admin.from_entry(cell_admin.to_entry(cell)))
def _resolve_cell_from_ldap(self, cellname): """Resolve Zookeeper connection sting from LDAP by cell name.""" # TODO: in case of invalid cell it will throw ldap exception. # need to standardize on ContextError raised lazily # on first connection attempt, and keep resolve # exception free. admin_cell = admin.Cell(self.ldap.conn) try: cell = admin_cell.get(cellname) zk_hostports = [ '%s:%s' % (master['hostname'], master['zk-client-port']) for master in cell['masters'] ] self.zk.url = 'zookeeper://%s@%s/treadmill/%s' % ( cell['username'], ','.join(zk_hostports), cellname) self.cell = cellname except ldap3.LDAPNoSuchObjectResult: exception = ContextError( 'Cell not defined in LDAP {}'.format(cellname)) _LOGGER.debug(str(exception)) raise exception return bool(self.zk.url)
def resolve(ctx, attr): """Resolve context attribute.""" if attr != 'zk_url': raise KeyError(attr) # TODO: in case of invalid cell it will throw ldap exception. # need to standardize on ContextError raised lazily # on first connection attempt, and keep resolve # exception free. try: admin_cell = admin.Cell(ctx.ldap.conn) cell = admin_cell.get(ctx.cell) zk_hostports = [ '%s:%s' % (master['hostname'], master['zk-client-port']) for master in cell['masters'] ] return 'zookeeper://%s@%s/treadmill/%s' % ( cell['username'], ','.join(zk_hostports), ctx.cell) except ldap_exceptions.LDAPNoSuchObjectResult: exception = context.ContextError('Cell not defined in LDAP {}'.format( ctx.cell)) _LOGGER.debug(str(exception)) raise exception
def sync_partitions(): """Syncs partitions to Zookeeper. """ _LOGGER.info('Sync: partitions.') zkclient = context.GLOBAL.zk.conn admin_cell = admin.Cell(context.GLOBAL.ldap.conn) partitions = admin_cell.partitions(context.GLOBAL.cell) zkclient.ensure_path(z.path.partition()) in_zk = zkclient.get_children(z.path.partition()) names = [partition['_id'] for partition in partitions] for extra in set(in_zk) - set(names): _LOGGER.debug('Delete: %s', extra) zkutils.ensure_deleted(zkclient, z.path.partition(extra)) # Add or update current partitions for partition in partitions: zkname = partition['_id'] if 'reboot-schedule' in partition: try: partition['reboot-schedule'] = utils.reboot_schedule( partition['reboot-schedule'] ) except ValueError: _LOGGER.info('Invalid reboot schedule, ignoring.') del partition['reboot-schedule'] if zkutils.put(zkclient, z.path.partition(zkname), partition, check_content=True): _LOGGER.info('Update: %s', zkname) else: _LOGGER.info('Up to date: %s', zkname)
def _admin_cell(): """Lazily return admin object.""" return admin.Cell(context.GLOBAL.ldap.conn)
def _list(): """Displays servers list.""" admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cells = admin_cell.list({}) cli.out(formatter(cells))
def create_zk(ec2_conn, ipa_client, master, subnet_id=None, ip_address=None, instance_type=None, instance_profile=None, image_id=None, disk=None): """ Create new Zookeeper """ ipa_domain = awscontext.GLOBAL.ipa_domain admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(context.GLOBAL.cell) data = cell['data'] if not image_id: image_id = data['image'] if not image_id.startswith('ami-'): image_id = ec2client.get_image(ec2_conn, owners=['self'], name=image_id)['ImageId'] # FIXME; subnet not unique among ZK, not AZ aware if not subnet_id: subnet_id = random.choice(data['subnets']) shortname = master['hostname'].replace('.' + context.GLOBAL.dns_domain, '') if not instance_type: instance_type = 'm5.large' if not instance_profile: instance_profile = 'zk-server' # Instance vars instance_vars = { 'instance_profile': instance_profile, 'treadmill_cell': context.GLOBAL.cell, 'treadmill_ldap': ','.join(context.GLOBAL.ldap.url), 'treadmill_ldap_suffix': context.GLOBAL.ldap_suffix, 'treadmill_dns_domain': context.GLOBAL.dns_domain, 'treadmill_isa': 'zookeeper', 'treadmill_profile': 'aws', 'treadmill_krb_realm': krb5.get_host_realm(sysinfo.hostname())[0], 'treadmill_zookeeper_myid': str(master['idx']), 'treadmill_zookeeper_admins': ','.join(set([cell['username'], data['aws_admin']])) } # Build user-data and start new instance create_host(ec2_conn=ec2_conn, ipa_client=ipa_client, image_id=image_id, count=1, domain=ipa_domain, secgroup_ids=data['secgroup'], instance_type=instance_type, subnet=subnet_id, disk=disk or 30, instance_vars=instance_vars, role='zookeeper', hostgroups=['zookeepers'], hostname=shortname, ip_address=ip_address) return master['hostname']
def create_n_servers(count, partition=None): """Create new servers in the cell.""" ipa_client = awscontext.GLOBAL.ipaclient ec2_conn = awscontext.GLOBAL.ec2 sts_conn = awscontext.GLOBAL.sts ipa_domain = awscontext.GLOBAL.ipa_domain admin_srv = admin.Server(context.GLOBAL.ldap.conn) admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(context.GLOBAL.cell) data = cell['data'] image_id = data['image'] if not image_id.startswith('ami-'): account = sts_conn.get_caller_identity().get('Account') image_id = ec2client.get_image(ec2_conn, owners=[account], name=image_id)['ImageId'] instance_type = data['size'] subnets = data['subnets'] secgroup_id = data['secgroup'] hostgroups = data['hostgroups'] instance_profile = data['instance_profile'] disk_size = int(data['disk_size']) hostname_template = '{}-{}-{}'.format(context.GLOBAL.cell, partition if partition else 'node', '{time}') instance_vars = { 'treadmill_cell': context.GLOBAL.cell, 'treadmill_ldap': ','.join(context.GLOBAL.ldap.url), 'treadmill_ldap_suffix': context.GLOBAL.ldap_suffix, 'treadmill_dns_domain': context.GLOBAL.dns_domain, 'treadmill_isa': 'node', 'treadmill_profile': 'aws', 'treadmill_krb_realm': krb5.get_host_realm(sysinfo.hostname())[0], } # FIXME: Add Partition: $partition to tags when Autoscaler is cell aware tags = [{'Key': 'Cell', 'Value': context.GLOBAL.cell}] key = None for idx in range(0, count): hostnames = hostmanager.create_host(ipa_client=ipa_client, ec2_conn=ec2_conn, image_id=image_id, count=1, disk=disk_size, domain=ipa_domain, key=key, secgroup_ids=secgroup_id, instance_type=instance_type, subnets=subnets, role='node', instance_vars=instance_vars, instance_profile=instance_profile, hostgroups=hostgroups, hostname=hostname_template, ip_address=None, eni=None, tags=tags) # Count is one, but it is more robust to treat it as list. for hostname in hostnames: print(hostname) attrs = {'cell': context.GLOBAL.cell, 'partition': partition} admin_srv.create(hostname, attrs)
def test_get(self): """Dummy test for treadmill.api.cell.get()""" cell_admin = admin.Cell(None) self.cell.get('some-cell') cell_admin.get.assert_called_with('some-cell')
def test_list(self): """Dummy test for treadmill.api.cell._list()""" self.cell.list() cell_admin = admin.Cell(None) self.assertTrue(cell_admin.list.called)
def _metadata(): """Returns check metadata.""" _meta = { 'index': 'name', 'query': 'select * from servers order by partition', 'checks': [{ 'description': 'Partitions capacity.', 'query': """ select name, partition, presence from servers where presence != 1 order by partition """, 'metric': """ select partition, count(*) as down from ({query}) group by partition """, 'alerts': [], }, { 'description': 'Topology syncronised.', 'query': """ select name, partition, in_zk, in_ldap from servers where in_zk == 0 or in_ldap == 0 order by partition """, 'metric': """ select count(*) as not_synced from ({query}) group by partition """, 'alerts': [ { 'description': 'Servers synced between LDAP and Zk', 'severity': 'error', 'threshold': { 'not_synced': 0 } }, ] }] } admin_cell = admin.Cell(context.GLOBAL.ldap.conn) cell = admin_cell.get(context.GLOBAL.cell) partitions = cell.get('partitions', [{'_id': '_default'}]) has_default = False for partition in partitions: name = partition['_id'] down_threshold = partition.get('down-threshold', 0) if name == '_default': has_default = True _meta['checks'][0]['alerts'].append({ 'description': 'Partition: {partition}', 'severity': 'error', 'match': { 'partition': name, }, 'threshold': { 'down': down_threshold, } }) if not has_default: _meta['checks'][0]['alerts'].append({ 'description': 'Partition: {partition}', 'severity': 'error', 'match': { 'partition': '_default' }, 'threshold': { 'down': 0, } }) return _meta