def _run_sync():
"""Sync Zookeeper with LDAP, runs with lock held."""
def match_appgroup(name, group):
"""Match if appgroup belongs to the cell."""
if context.GLOBAL.cell in group.get('cells', []):
return name
else:
return None
while True:
# Sync app groups
admin_app_group = admin.AppGroup(context.GLOBAL.ldap.conn)
app_groups = admin_app_group.list({})
_sync_collection(context.GLOBAL.zk.conn,
app_groups, z.path.appgroup(), match_appgroup)
# Sync allocations.
admin_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
allocations = admin_alloc.list({'cell': context.GLOBAL.cell})
_sync_allocations(context.GLOBAL.zk.conn,
allocations)
# Servers - because they can have custom topology - are loaded
# from the plugin.
try:
servers_plugin = importlib.import_module(
'treadmill.plugins.sproc.servers')
servers_plugin.init()
except ImportError as err:
_LOGGER.warn('Unable to load treadmill.plugins.sproc.servers: '
'%s', err)
time.sleep(60)
def reserve(allocation, cell, memory, cpu, disk, rank, max_utilization,
features):
"""Reserve capacity on a given cell."""
admin_cell_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
data = {}
if memory:
data['memory'] = memory
if cpu:
data['cpu'] = cpu
if disk:
data['disk'] = disk
if rank is not None:
data['rank'] = rank
if max_utilization is not None:
data['max_utilization'] = max_utilization
if features:
data['features'] = features
try:
admin_cell_alloc.create([cell, allocation], data)
except ldap3.LDAPEntryAlreadyExistsResult:
admin_cell_alloc.update([cell, allocation], data)
try:
admin_alloc = admin.Allocation(context.GLOBAL.ldap.conn)
cli.out(formatter(admin_alloc.get(allocation)))
except ldap3.LDAPNoSuchObjectResult:
click.echo('Allocation does not exist: %s' % allocation, err=True)
def sync_allocations():
"""Syncronize allocations.
"""
_LOGGER.info('Sync allocations.')
zkclient = context.GLOBAL.zk.conn
admin_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
allocations = admin_alloc.list({'cell': context.GLOBAL.cell})
filtered = []
for alloc in allocations:
_LOGGER.info('Sync allocation: %s', alloc)
name, _cell = alloc['_id'].rsplit('/', 1)
alloc['name'] = name
filtered.append(alloc)
masterapi.update_allocations(zkclient, filtered)
def test_reservation(self):
"""Dummy test for treadmill.api.allocation.create()"""
alloc_admin = admin.CellAllocation(None)
self.alloc.reservation.create('tenant/alloc/cellname', {
'memory': '1G',
'cpu': '100%',
'disk': '2G',
'partition': None
})
alloc_admin.create.assert_called_with(
['cellname', 'tenant/alloc'],
{
'disk': '2G',
'partition': None,
'cpu': '100%',
'rank': 100,
'memory': '1G'
},
)
def assign(allocation, cell, priority, pattern, delete):
"""Manage application assignments"""
admin_cell_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
assignment = {'pattern': pattern, 'priority': priority}
if delete:
assignment['_delete'] = True
data = {'assignments': [assignment]}
if delete:
admin_cell_alloc.update([cell, allocation], data)
else:
try:
admin_cell_alloc.create([cell, allocation], data)
except ldap_exceptions.LDAPEntryAlreadyExistsResult:
admin_cell_alloc.update([cell, allocation], data)
try:
admin_alloc = admin.Allocation(context.GLOBAL.ldap.conn)
cli.out(formatter(admin_alloc.get(allocation)))
except ldap_exceptions.LDAPNoSuchObjectResult:
click.echo('Allocation does not exist: %s' % allocation, err=True)
def _run_sync():
"""Sync Zookeeper with LDAP, runs with lock held.
"""
while True:
# Sync app groups
admin_app_group = admin.AppGroup(context.GLOBAL.ldap.conn)
app_groups = admin_app_group.list({})
_sync_collection(context.GLOBAL.zk.conn, app_groups, z.path.appgroup(),
_match_appgroup)
# Sync partitions
admin_cell = admin.Cell(context.GLOBAL.ldap.conn)
partitions = admin_cell.partitions(context.GLOBAL.cell)
_sync_partitions(context.GLOBAL.zk.conn, partitions)
# Sync allocations.
admin_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
allocations = admin_alloc.list({'cell': context.GLOBAL.cell})
_sync_allocations(context.GLOBAL.zk.conn, allocations)
# Global servers
admin_srv = admin.Server(context.GLOBAL.ldap.conn)
global_servers = admin_srv.list({})
zkutils.ensure_exists(
context.GLOBAL.zk.conn,
z.path.globals('servers'),
data=[server['_id'] for server in global_servers])
# Servers - because they can have custom topology - are loaded
# from the plugin.
try:
servers_plugin = importlib.import_module(
'treadmill.plugins.sproc.servers')
servers_plugin.init()
except ImportError as err:
_LOGGER.warning(
'Unable to load treadmill.plugins.sproc.servers: %s', err)
time.sleep(60)
def reserve(allocation, cell, memory, cpu, disk, rank, rank_adjustment,
max_utilization, traits, partition, delete):
"""Reserve capacity on a given cell"""
admin_cell_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
if delete:
admin_cell_alloc.delete([cell, allocation])
return
data = {}
if memory:
data['memory'] = memory
if cpu:
data['cpu'] = cpu
if disk:
data['disk'] = disk
if rank is not None:
data['rank'] = rank
if rank_adjustment is not None:
data['rank_adjustment'] = rank_adjustment
if max_utilization is not None:
data['max_utilization'] = max_utilization
if traits:
data['traits'] = cli.combine(traits)
if partition:
if partition == '-':
partition = None
data['partition'] = partition
try:
admin_cell_alloc.create([cell, allocation], data)
except ldap_exceptions.LDAPEntryAlreadyExistsResult:
admin_cell_alloc.update([cell, allocation], data)
try:
admin_alloc = admin.Allocation(context.GLOBAL.ldap.conn)
cli.out(formatter(admin_alloc.get(allocation, dirty=True)))
except ldap_exceptions.LDAPNoSuchObjectResult:
click.echo('Allocation does not exist: %s' % allocation, err=True)
def assign(allocation, cell, priority, pattern, delete):
"""Manage application assignments"""
admin_cell_alloc = admin.CellAllocation(context.GLOBAL.ldap.conn)
try:
cell_alloc = admin_cell_alloc.get([cell, allocation])
except ldap_exceptions.LDAPNoSuchObjectResult:
cell_alloc = {}
assignments = cell_alloc.get('assignments', [])
if delete:
new_assigments = [
assignment for assignment in assignments
if assignment['pattern'] != pattern
]
assignments = new_assigments
else:
assignment_attrs = {'priority': priority}
for assignment in assignments:
if assignment['pattern'] == pattern:
assignment.update(assignment_attrs)
break
else:
assignments.append({'pattern': pattern, 'priority': priority})
cell_alloc_attrs = {'assignments': assignments}
if cell_alloc:
admin_cell_alloc.update([cell, allocation], cell_alloc_attrs)
else:
admin_cell_alloc.create([cell, allocation], cell_alloc_attrs)
try:
admin_alloc = admin.Allocation(context.GLOBAL.ldap.conn)
cli.out(formatter(admin_alloc.get(allocation, dirty=True)))
except ldap_exceptions.LDAPNoSuchObjectResult:
click.echo('Allocation does not exist: %s' % allocation, err=True)
def setUp(self):
self.cell_alloc = admin.CellAllocation(
admin.Admin(None, 'dc=xx,dc=com'))
def _admin_cell_alloc():
"""Lazily return admin cell allocation object.
"""
return admin.CellAllocation(context.GLOBAL.ldap.conn)
