def get_active_allocations():
auth_session = get_auth_session()
aclient = allocation_client.Client('1', session=auth_session)
active_allocations = {}
# All approved allocations
allocations = aclient.allocations.list(parent_request__isnull=True)
for allocation in allocations:
if (allocation.status == states.DELETED
or allocation.status == states.SUBMITTED):
continue
elif allocation.status != states.APPROVED:
try:
allocation = aclient.allocations.get_last_approved(
parent_request=allocation.id)
except exceptions.AllocationDoesNotExist:
continue
if not allocation.project_id:
continue
active_allocations[allocation.project_id] = allocation
return active_allocations
def stat(host=None,
last_changed=None,
availability_zone=None,
aggregate=None,
status=None,
project=None,
allocation_home=None,
exclude_availability_zone=None,
exclude_host=None,
exclude_aggregate=None,
detail=None):
"""Gather statistic of all nova instances with given parameters."""
session = get_session()
nova = nova_client.Client(2, session=session)
glance = glance_client.Client(2, session=session)
keystone = keystone_client.Client(session=session)
allocation = allocation_client.Client(1, session=session)
instances = _list_instances(nova, glance, keystone, allocation, aggregate,
availability_zone, host, status, project,
allocation_home, exclude_availability_zone,
exclude_host, exclude_aggregate)
# Summary table
table = PrettyTable(['Name', 'Value'])
data = {'instances': 0, 'vcpus': 0, 'ram': 0}
table.align = 'l'
projects_counter = Counter()
# Detail table
dt = PrettyTable(['Project', 'Instances', 'vCPUs', 'RAM'])
projects = defaultdict(lambda: defaultdict(int))
dt.align = 'l'
for ins in instances:
project_name = ins._info['project'].name
projects[project_name]['instances'] += 1
data['instances'] += 1
projects_counter[ins._info['project'].name] += 1
if not ins._info['flavor']['vcpus']:
continue
data['vcpus'] += int(ins._info['flavor']['vcpus'])
data['ram'] += int(ins._info['flavor']['ram'])
projects[project_name]['vcpus'] += int(ins._info['flavor']['vcpus'])
projects[project_name]['ram'] += int(ins._info['flavor']['ram'])
data['common'] = ',\n'.join('%s: %d' % (k, v)
for k, v in projects_counter.most_common(3))
# Convert the data to bytes for humanization
data['ram'] = data['ram'] * 1024 * 1024
table.add_row(['Total instances', data['instances']])
table.add_row(['vCPUs used', data['vcpus']])
table.add_row(['RAM used', humanize.naturalsize(data['ram'], binary=True)])
table.add_row(['Total projects affected', len(projects_counter.keys())])
table.add_row(['Top projects affected', data['common']])
click.echo(table)
if detail:
for name, p in projects.items():
dt.add_row([
name, p['instances'], p['vcpus'],
humanize.naturalsize(p['ram'] * 1024 * 1024, binary=True)
])
click.echo(dt)
def list(host=None,
last_changed=None,
availability_zone=None,
column=None,
aggregate=None,
status=None,
sort_by=None,
project=None,
allocation_home=None,
exclude_availability_zone=None,
exclude_host=None,
exclude_aggregate=None):
"""List all nova instances with given parameters."""
session = get_session()
nova = nova_client.Client(2, session=session)
glance = glance_client.Client(2, session=session)
keystone = keystone_client.Client(session=session)
allocation = allocation_client.Client(1, session=session)
instances = _list_instances(nova, glance, keystone, allocation, aggregate,
availability_zone, host, status, project,
allocation_home, exclude_availability_zone,
exclude_host, exclude_aggregate)
# INSTAN = instances
# embed()
if not column:
sort_by = 'OS-EXT-SRV-ATTR:host'
_render_table_instances(instances, column, sort_by)
def __init__(self, conf):
super(AllocationPollsterBase, self).__init__(conf)
creds = conf.service_credentials
self.client = client.Client(
version='1',
session=keystone_client.get_session(conf),
region_name=creds.region_name,
interface=creds.interface,
)
def setup_allocations(self):
if not self.allocations:
loader = loading.get_plugin_loader('password')
username = os.environ.get('OS_USERNAME')
password = os.environ.get('OS_PASSWORD')
auth_url = os.environ.get('OS_AUTH_URL')
project_name = os.environ.get('OS_TENANT_NAME')
auth = loader.load_from_options(auth_url=auth_url,
username=username,
password=password,
project_name=project_name,
user_domain_id='default',
project_domain_id='default')
sess = session.Session(auth=auth)
self.allocations = client.Client(1, session=sess)
def list_user_projects(email, conn):
"""
user id --> all projects (including closed ones) --> flavor
:param email: email as user ID
:param conn: openstack connection
:return:
"""
allocation_client = allocationclient.Client(1, session=conn.session)
# step 1: get the user ID from email
user = conn.get_user(email)
if user is None:
print('can not find user')
return
# step 2: get all the projects of a user
roles = conn.list_role_assignments({'user': user.id})
all_projects = [r.project for r in roles]
# print(all_projects)
# step 3: get all auckland GPU flavors
akl_gpu_flavors = conn.search_flavors('akl.gpu*', get_extra=False)
user_allocations = []
for f in akl_gpu_flavors:
for access in conn.list_flavor_access(f.id):
# print(access)
if access.project_id in all_projects:
alloc = fetch_project_info(access.project_id,
allocation_client)
if alloc:
user_allocations.append(alloc)
# print(f'{alloc.project_name}({alloc.project_id}): {alloc.start_date} -- {alloc.end_date}')
# output
x = PrettyTable()
x.field_names = [
'project_name', 'project_id', 'status', 'start_date', 'end_date'
]
for a in user_allocations:
x.add_row([
a.project_name, a.project_id, a.status_display, a.start_date,
a.end_date
])
print(x)
def update_gpu_db(osc_conn, db_conn, gpudb_conn):
"""
save current metadata on current instances that are using a GPU
:param osc_conn:
:param gpudb_conn:
:return:
"""
allocation_client = allocationclient.Client(1, session=osc_conn.session)
devices = fetch_pci_device_from_db(db_conn)
all_project_ids = [
d['project_id'] for d in devices if d['project_id'] is not None
]
# process assigned projects but not VM running
# assumption, 1 project can only have 1 GPU instance
akl_gpu_flavor = osc_conn.search_flavors('akl.gpu*', get_extra=False)
for f in akl_gpu_flavor:
all_access = osc_conn.list_flavor_access(f.id)
for a in all_access:
# test if it already has an instance
if a.project_id not in all_project_ids:
# check if the project is still active
project = fetch_project_info(a.project_id, allocation_client)
if project is None: # or project.end_date < str(date.today()):
continue
# we need to fix the project
#for project in fetch_project_info:
# get extra properties and project access info
detail = osc_conn.get_flavor_by_id(f.id, get_extra=True)
gpu_model = detail.extra_specs['pci_passthrough:alias'].split(
':')[0]
# print(f"{a.project_id} no instance, but has flavor {f.name}, GPU={gpu_model}")
# find an available model and change the status
for d in devices:
if d['label'] == gpu_model and d['status'] == 'available':
d['status'] = 'reserved'
d['project_id'] = a.project_id
break
current_gpu_node_list = fetch_gpu_nodes(gpudb_conn)
#First we set tmp_active to 0 for all entries,
#then for every host,device pair, we set it to 1
#At the end, we set active to tmp_active for all entries
cursor = gpudb_conn.cursor()
try:
cursor.execute('UPDATE gpu_nodes SET tmp_active = 0')
gpudb_conn.commit()
except pymysql.Error as e:
print "DB clear gpu_nodes.tmp_active: ", e
# update project_end date and contact
for d in devices:
# get project allocation info
alloc = None
if d['project_id'] is not None and len(d['project_id']) > 0:
alloc = fetch_project_info(d['project_id'], allocation_client)
#if alloc is not None:
# print alloc.__dict__
if d['display_name'] is None and alloc is not None:
d['display_name'] = alloc.project_name
d['project_name'] = 'Auckland-CeR' if alloc is None else alloc.project_name
d['start_date'] = '2017-07-04' if alloc is None else alloc.start_date
d['end_date'] = '9999-12-31' if alloc is None else alloc.end_date
if d['launched_at'] is None:
d['launched_at'] = d['start_date']
if d['terminated_at'] is None:
d['terminated_at'] = d['end_date']
d['contact'] = None if alloc is None else alloc.contact_email
d['ip'] = find_ip(osc_conn, d['instance_uuid'])
d['host'] = d['host'].replace("ntr-", "")
d['host'] = d['host'].replace("akld2", "")
statement1 = '''
INSERT INTO ip2project (ip, project_name, project_uuid, start_date, end_date, email, instance_uuid, instance_name, instance_launched_at, instance_terminated_at )
VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON DUPLICATE KEY UPDATE instance_terminated_at = %s, end_date = %s, final = 0
'''
statement2 = '''
INSERT INTO gpu_nodes ( hypervisor, gpu_type, pci_id, tmp_active ) VALUES ( %s, %s, %s, %s ) ON DUPLICATE KEY UPDATE tmp_active = %s
'''
statement3 = '''
INSERT into ip2project_gpu_nodes (ip2project_id, gpu_node_id) values
((select ip2project.id from ip2project where ip = %s and project_uuid = %s and instance_uuid = %s), (select gpu_nodes.id from gpu_nodes where hypervisor = %s and pci_id = %s)) ON DUPLICATE KEY UPDATE ip2project_id = ip2project_id
'''
if current_gpu_node_list is not None:
key = d['host'] + ' ' + d['dev_id']
if key in current_gpu_node_list:
current_gpu_node_list[key]['present'] = True
cursor = gpudb_conn.cursor()
try:
cursor.execute(statement2,
(d['host'], d['label'], d['dev_id'], '1', '1'))
gpudb_conn.commit()
except pymysql.Error as e:
print "DB Update gpu_nodes: ", e
if d['instance_uuid'] is not None and d[
'ip'] is not None and d['ip'] != '':
cursor = gpudb_conn.cursor()
try:
cursor.execute(
statement1,
(d['ip'], d['project_name'], d['project_id'],
d['start_date'], d['end_date'], d['contact'],
d['instance_uuid'], d['display_name'], d['launched_at'],
d['terminated_at'], d['terminated_at'], d['end_date']))
cursor.execute(statement3,
(d['ip'], d['project_id'], d['instance_uuid'],
d['host'], d['dev_id']))
gpudb_conn.commit()
except pymysql.Error as e:
print "DB Update gpu_nodes: ", e
gpudb_conn.rollback()
elif d['project_id'] is not None:
statement4 = '''
INSERT INTO gpu_booking (project_name, project_uuid, booking_start_date, booking_end_date, email, gpu_type, count )
VALUES(%s,%s,%s,%s,%s,%s,%s)
ON DUPLICATE KEY UPDATE booking_end_date = %s
'''
cursor = gpudb_conn.cursor()
try:
cursor.execute(statement4,
(d['project_name'], d['project_id'],
d['start_date'], d['end_date'], d['contact'],
d['label'], '1', d['end_date']))
gpudb_conn.commit()
except pymysql.Error as e:
print "DB Update gpu_booking: ", e
gpudb_conn.rollback()
#At the end, we set active to tmp_active for all entries
cursor = gpudb_conn.cursor()
try:
cursor.execute('UPDATE gpu_nodes SET active = tmp_active')
gpudb_conn.commit()
except pymysql.Error as e:
print "DB set gpu_nodes.active: ", e
def list_gpus(osc_conn, db_conn):
"""
list all GPUs and their utilisation
:param osc_conn:
:param db_conn:
:return:
"""
allocation_client = allocationclient.Client(1, session=osc_conn.session)
devices = fetch_pci_device_from_db(db_conn)
all_project_ids = [
d['project_id'] for d in devices if d['project_id'] is not None
]
# process assigned projects but not VM running
# assumption, 1 project can only have 1 GPU instance
akl_gpu_flavor = osc_conn.search_flavors('akl.gpu*', get_extra=False)
for f in akl_gpu_flavor:
all_access = osc_conn.list_flavor_access(f.id)
for a in all_access:
# test if it already has an instance
if a.project_id not in all_project_ids:
# check if the project is still active
project = fetch_project_info(a.project_id, allocation_client)
if project is None: # or project.end_date < str(date.today()):
continue
# we need to fix the project
#for project in fetch_project_info:
# get extra properties and project access info
detail = osc_conn.get_flavor_by_id(f.id, get_extra=True)
gpu_model = detail.extra_specs['pci_passthrough:alias'].split(
':')[0]
# print(f"{a.project_id} no instance, but has flavor {f.name}, GPU={gpu_model}")
# find an available model and change the status
for d in devices:
if d['label'] == gpu_model and d['status'] == 'available':
d['status'] = 'reserved'
d['project_id'] = a.project_id
break
# update project_end date and contact
for d in devices:
# get project allocation info
alloc = None
if d['project_id'] is not None and len(d['project_id']) > 0:
alloc = fetch_project_info(d['project_id'], allocation_client)
#if alloc is not None:
# print alloc.__dict__
if d['display_name'] is None and alloc is not None:
d['display_name'] = alloc.project_name
d['project_name'] = 'Auckland-CeR' if alloc is None else alloc.project_name
d['start_date'] = 'NA' if alloc is None else alloc.start_date
d['end_date'] = 'NA' if alloc is None else alloc.end_date
d['contact'] = 'NA' if alloc is None else alloc.contact_email
d['ip'] = find_ip(osc_conn, d['instance_uuid'])
d['host'] = d['host'].replace("ntr-", "")
d['host'] = d['host'].replace("akld2", "")
# output
x = PrettyTable()
x.field_names = [
'host', 'label', 'status', 'instance_uuid', 'display_name',
'project_name', 'project_id', 'start_date', 'end_date', 'contact',
'dev_id', 'ip', 'launched_at', 'terminated_at'
]
for d in devices:
row_data = []
for f in x.field_names:
row_data.append(d[f])
x.add_row(row_data)
print(x)