def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.get_glance_client(kc)
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info["%s network" % network_label] = ", ".join(address_list)
flavor = info.get("flavor", {})
flavor_id = flavor.get("id", "")
try:
info["flavor"] = "%s (%s)" % (nova.get_flavor(nc, flavor_id).name, flavor_id)
except Exception:
info["flavor"] = "%s (%s)" % ("Flavor not found", flavor_id)
# Image
image = info.get("image", {})
if image:
image_id = image.get("id", "")
try:
img = gc.images.get(image_id)
nectar_build = img.properties.get("nectar_build", "N/A")
info["image"] = "%s (%s, NeCTAR Build %s)" % (img.name, img.id, nectar_build)
except Exception:
info["image"] = "Image not found (%s)" % image_id
else: # Booted from volume
info["image"] = "Attempt to boot from volume - no image supplied"
# Tenant
tenant_id = info.get("tenant_id")
if tenant_id:
try:
tenant = keystone.get_tenant(kc, tenant_id)
info["tenant_id"] = "%s (%s)" % (tenant.name, tenant.id)
except Exception:
pass
# User
user_id = info.get("user_id")
if user_id:
try:
user = keystone.get_user(kc, user_id)
info["user_id"] = "%s (%s)" % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop("links", None)
info.pop("addresses", None)
info.pop("hostId", None)
info.pop("security_groups", None)
return _format_instance(info, style=style)
def compare_quotas(name_or_id=None):
"""Compare the allocation and quota information for a tenant
"""
if name_or_id == None:
print 'A tenant name or id is required'
return
keystone_api = hm_keystone.client_session(version=3)
try:
tenant = hm_keystone.get_tenant(keystone_api, name_or_id)
except:
print 'Tenant {0} not found in keystone'.format(name_or_id)
return
nova_api = hm_nova.client()
quotas = nova_api.quotas.get(tenant.id)
print 'nova quotas: instances {0}, cores {1}, ram {2}'.format(
quotas.instances, quotas.cores, quotas.ram / 1024)
usage = _get_usage(nova_api, _get_flavor_map(nova_api), tenant.id)
print 'nova usage: instances {0}, cores {1}, ram {2}'.format(
usage['instances'], usage['vcpus'], usage['ram'] / 1024)
allocations_api = NectarApiSession()
allocations = allocations_api.get_allocations();
tenant_allocations = filter(lambda x: x['tenant_uuid'] == tenant.id and \
(x['status'] == 'A' or x['status'] == 'X'),
allocations)
if len(tenant_allocations) == 0:
print 'No approved allocation records for tenant {0} / {1}'.format(
tenant.id, tenant.name)
return
tenant_allocations.sort(key=lambda alloc: alloc['modified_time'])
current_allocation = tenant_allocations[-1]
format = '{0} mismatch: allocated {1}, nova {2}, used {3}'
if current_allocation['instance_quota'] != quotas.instances:
print format.format('Instance quota',
current_allocation['instance_quota'],
quotas.instances,
usage['instances'])
if current_allocation['core_quota'] != quotas.cores:
print format.format('VCPU quota',
current_allocation['core_quota'],
quotas.cores,
usage['vcpus'])
if current_allocation['ram_quota'] * 1024 != quotas.ram:
print format.format('RAM quota',
current_allocation['ram_quota'] * 1024,
quotas.ram,
usage['ram'])
def get_ticket_recipients(instance):
"""Build a list of email addresses"""
email_addresses = []
kc = keystone.client()
user = keystone.get_user(kc, instance.user_id)
if user.email:
email_addresses.append(user.email)
# Add tenant members to ticket recipient list
tenant = keystone.get_tenant(kc, instance.tenant_id)
for user in tenant.list_users():
roles = [r.name for r in user.list_roles(tenant)]
if 'TenantManager' in roles:
email_addresses.append(user.email)
return email_addresses
def purge_project(tenant_name, dry_run=True):
"""Purge resources and disable a given project """
if not spawn.find_executable('ospurge'):
error('ospurge not found in path. Please ensure it is installed.')
username, password = get_creds()
if dry_run:
print("Running in dry-run mode")
ks_client = keystone.client()
tenant = keystone.get_tenant(ks_client, tenant_name)
tenant_member_role = ks_client.roles.find(name='Member')
ospurge_user = keystone.get_user(ks_client, username)
if not tenant.enabled:
print("Enabling project for purge")
tenant.update(enabled=True)
if not is_in_project(tenant, ospurge_user):
print("Adding ospurge user to project")
tenant.add_user(ospurge_user, tenant_member_role)
if dry_run:
run_opt = '--dry-run'
else:
run_opt = '--verbose'
cmd = ("ospurge --dont-delete-project --own-project --username {username} "
"--password {password} --admin-project {tenant} {run_opt}"
"".format(username=username, password=password,
tenant=tenant.name, run_opt=run_opt))
print("Running: {}".format(cmd.replace(password, 'xxxx')))
run(cmd)
if is_in_project(tenant, ospurge_user):
print("Removing ospurge user from project")
tenant.remove_user(ospurge_user, tenant_member_role)
if tenant.enabled:
if dry_run and tenant.enabled:
print("Not disabling project due to dry-run")
else:
print("Disabling project")
tenant.update(enabled=False)
keystone.set_project_metadata(tenant_name,
'ospurge_date',
str(datetime.datetime.now()))
def get_images_tenant(tenant_id_or_name, tenant_type):
"""fetch tenant id from config file"""
if tenant_id_or_name is None:
msg = " ".join(("No tenant set.", "Please set tenant in",
"[cfg:hivemind_contrib.glance.%s]" % tenant_type))
error(msg)
try:
ks_client = keystone.client()
tenant = keystone.get_tenant(ks_client, tenant_id_or_name)
except ks_exc.NotFound:
raise error("Tenant {} not found. Check your settings."
.format(tenant_id_or_name))
except ks_exc.Forbidden:
raise error("Permission denied. Check you're using admin credentials.")
except Exception as e:
raise error(e)
return tenant
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd = get_freshdesk_client()
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
tenant = keystone.get_tenant(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email_addresses = get_ticket_recipients(instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {}'.format(instance_id)
body = '<br />\n'.join([
'Dear NeCTAR Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance.id),
'in the project <b>{}</b>'.format(tenant.name),
'created by <b>{}</b>'.format(user.email),
'has been involved in a security incident.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'NeCTAR Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {}'.format(email_addresses))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_ticket(
description=body,
subject=subject,
email='*****@*****.**',
cc_emails=email_addresses,
priority=4,
status=6,
tags=['security'])
ticket_id = ticket.id
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(fd.domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
print('Would update ticket with action')
else:
# Pause and lock
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Add reply to user
email_addresses = get_ticket_recipients(instance)
print('Replying to ticket with action details')
action = 'Instance <b>{} ({})</b> has been <b>paused and locked</b> '\
'pending further investigation'\
.format(instance.name, instance_id)
fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
