def grant_access(user):
"""Grant the user access to reports.rc.nectar.org.au.
"""
ksclient = keystone.client()
user = keystone.get_user(ksclient, user)
keystone.add_project_roles(user.default_project_id, user.id, [ROLE])
keystone.print_members(ksclient, user.default_project_id)
def _populate_project_dict(instances):
session = keystone.get_session()
ksclient = keystone.client(session=session)
project = collections.defaultdict(dict)
for instance in instances:
if instance['project_name'] in project.keys():
project[instance['project_name']]['instances'].append(instance)
else:
cclist = []
for role in ['TenantManager', 'Member']:
members = keystone.list_members(instance['project'], role)
if not members:
continue
for uid in members.keys():
user = keystone.get_user(ksclient, uid, use_cache=True)
if getattr(user, 'enabled', None) and \
getattr(user, 'email', None):
cclist.append(user.email)
# rule out the projects where has no valid recipients(tempest. etc)
if cclist:
project[instance['project_name']] = {'instances': [instance]}
project[instance['project_name']].update(
{'recipients': cclist})
return project
def revoke_access(user):
"""Revoke access to reports.rc.nectar.org.au for the user.
"""
ksclient = keystone.client()
user = keystone.get_user(ksclient, user)
keystone.remove_project_roles(user.default_project_id, user.id, [ROLE])
keystone.print_members(ksclient, user.default_project_id)
def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.get_glance_client(kc)
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info["%s network" % network_label] = ", ".join(address_list)
flavor = info.get("flavor", {})
flavor_id = flavor.get("id", "")
try:
info["flavor"] = "%s (%s)" % (nova.get_flavor(nc, flavor_id).name, flavor_id)
except Exception:
info["flavor"] = "%s (%s)" % ("Flavor not found", flavor_id)
# Image
image = info.get("image", {})
if image:
image_id = image.get("id", "")
try:
img = gc.images.get(image_id)
nectar_build = img.properties.get("nectar_build", "N/A")
info["image"] = "%s (%s, NeCTAR Build %s)" % (img.name, img.id, nectar_build)
except Exception:
info["image"] = "Image not found (%s)" % image_id
else: # Booted from volume
info["image"] = "Attempt to boot from volume - no image supplied"
# Tenant
tenant_id = info.get("tenant_id")
if tenant_id:
try:
tenant = keystone.get_tenant(kc, tenant_id)
info["tenant_id"] = "%s (%s)" % (tenant.name, tenant.id)
except Exception:
pass
# User
user_id = info.get("user_id")
if user_id:
try:
user = keystone.get_user(kc, user_id)
info["user_id"] = "%s (%s)" % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop("links", None)
info.pop("addresses", None)
info.pop("hostId", None)
info.pop("security_groups", None)
return _format_instance(info, style=style)
def get_tenant_managers_emails(kc, instance):
"""Build a list of email addresses"""
email_addresses = []
project = keystone.get_project(kc, instance.tenant_id)
role = kc.roles.find(name='TenantManager')
ras = kc.role_assignments.list(project=project, role=role,
include_names=True)
for ra in ras:
u = keystone.get_user(kc, ra.user['id'])
email_addresses.append(u.email)
return email_addresses
def extract_server_info(server, ksclient):
server_info = collections.defaultdict(dict)
try:
server_info['id'] = server.id
server_info['name'] = server.name
server_info['status'] = server.status
server_info['flavor'] = server.flavor['id']
server_info['host'] = getattr(server, "OS-EXT-SRV-ATTR:host")
server_info['zone'] = getattr(server, "OS-EXT-AZ:availability_zone")
# handle vms which are not booted from glance images
server_image = getattr(server, "image", None)
if server_image:
server_info['image'] = server_image.get("id", None)
else:
server_info['image'] = None
# handle some tier2 services which using "global" service user/project
if server.metadata and 'user_id' in server.metadata.keys()\
and 'project_id' in server.metadata.keys():
server_info['user'] = server.metadata['user_id']
server_info['project'] = server.metadata['project_id']
else:
server_info['user'] = server.user_id
server_info['project'] = server.tenant_id
server_info['addresses'] = _extract_ip(server)
server_info['project_name'] = keystone.get_project(
ksclient, server_info['project'], use_cache=True).name
user = keystone.get_user(
ksclient, server_info['user'], use_cache=True)
# handle instaces created by jenkins/tempest and users without fullname
# set disabled user's email/fullname as None as it should be ruled out
if not user.enabled:
server_info['email'], server_info['fullname'] = None, None
elif getattr(user, 'email', None):
server_info['email'], server_info['fullname']\
= user.email, getattr(user, 'full_name', None)
else:
server_info['email'], server_info['fullname']\
= user.name, None
except KeyError as e:
raise type(e)(e.message + ' missing in context: %s' % server.to_dict())
return server_info
def get_ticket_recipients(instance):
"""Build a list of email addresses"""
email_addresses = []
kc = keystone.client()
user = keystone.get_user(kc, instance.user_id)
if user.email:
email_addresses.append(user.email)
# Add tenant members to ticket recipient list
tenant = keystone.get_tenant(kc, instance.tenant_id)
for user in tenant.list_users():
roles = [r.name for r in user.list_roles(tenant)]
if 'TenantManager' in roles:
email_addresses.append(user.email)
return email_addresses
def purge_project(tenant_name, dry_run=True):
"""Purge resources and disable a given project """
if not spawn.find_executable('ospurge'):
error('ospurge not found in path. Please ensure it is installed.')
username, password = get_creds()
if dry_run:
print("Running in dry-run mode")
ks_client = keystone.client()
tenant = keystone.get_tenant(ks_client, tenant_name)
tenant_member_role = ks_client.roles.find(name='Member')
ospurge_user = keystone.get_user(ks_client, username)
if not tenant.enabled:
print("Enabling project for purge")
tenant.update(enabled=True)
if not is_in_project(tenant, ospurge_user):
print("Adding ospurge user to project")
tenant.add_user(ospurge_user, tenant_member_role)
if dry_run:
run_opt = '--dry-run'
else:
run_opt = '--verbose'
cmd = ("ospurge --dont-delete-project --own-project --username {username} "
"--password {password} --admin-project {tenant} {run_opt}"
"".format(username=username, password=password,
tenant=tenant.name, run_opt=run_opt))
print("Running: {}".format(cmd.replace(password, 'xxxx')))
run(cmd)
if is_in_project(tenant, ospurge_user):
print("Removing ospurge user from project")
tenant.remove_user(ospurge_user, tenant_member_role)
if tenant.enabled:
if dry_run and tenant.enabled:
print("Not disabling project due to dry-run")
else:
print("Disabling project")
tenant.update(enabled=False)
keystone.set_project_metadata(tenant_name,
'ospurge_date',
str(datetime.datetime.now()))
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd = get_freshdesk_client()
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
tenant = keystone.get_tenant(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email_addresses = get_ticket_recipients(instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {}'.format(instance_id)
body = '<br />\n'.join([
'Dear NeCTAR Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance.id),
'in the project <b>{}</b>'.format(tenant.name),
'created by <b>{}</b>'.format(user.email),
'has been involved in a security incident.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'NeCTAR Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {}'.format(email_addresses))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_ticket(
description=body,
subject=subject,
email='*****@*****.**',
cc_emails=email_addresses,
priority=4,
status=6,
tags=['security'])
ticket_id = ticket.id
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(fd.domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
print('Would update ticket with action')
else:
# Pause and lock
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Add reply to user
email_addresses = get_ticket_recipients(instance)
print('Replying to ticket with action details')
action = 'Instance <b>{} ({})</b> has been <b>paused and locked</b> '\
'pending further investigation'\
.format(instance.name, instance_id)
fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd_config = get_freshdesk_config()
fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key'])
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
# Pause and lock instance
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
else:
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Process ticket
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
project = keystone.get_project(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email = user.email or '*****@*****.**'
name = getattr(user, 'full_name', email)
cc_emails = get_tenant_managers_emails(kc, instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {} ({})'.format(
instance.name, instance_id)
body = '<br />\n'.join([
'Dear Nectar Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance_id),
'in the project <b>{}</b>'.format(project.name),
'created by <b>{}</b>'.format(email),
'has been involved in a security incident, and has been locked.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'Nectar Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {} <{}>'.format(name, email))
print(' CC: {}'.format(', '.join(cc_emails)))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_outbound_email(
name=name,
description=body,
subject=subject,
email=email,
cc_emails=cc_emails,
email_config_id=int(fd_config['email_config_id']),
group_id=int(fd_config['group_id']),
priority=4,
status=2,
tags=['security'])
ticket_id = ticket.id
# Use friendly domain name if using prod
if fd.domain == 'dhdnectar.freshdesk.com':
domain = 'support.ehelp.edu.au'
else:
domain = fd.domain
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)
def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info['%s network' % network_label] = ', '.join(address_list)
flavor = info.get('flavor', {})
flavor_id = flavor.get('id', '')
try:
info['flavor'] = '%s (%s)' % (nova.get_flavor(nc, flavor_id).name,
flavor_id)
except Exception:
info['flavor'] = '%s (%s)' % ("Flavor not found", flavor_id)
# Image
image = info.get('image', {})
if image:
image_id = image.get('id', '')
try:
img = gc.images.get(image_id)
nectar_build = img.get('nectar_build', 'N/A')
info['image'] = ('%s (%s, NeCTAR Build %s)'
% (img.name, img.id, nectar_build))
except Exception:
info['image'] = 'Image not found (%s)' % image_id
else: # Booted from volume
info['image'] = "Attempt to boot from volume - no image supplied"
# Tenant
project_id = info.get('tenant_id')
if project_id:
try:
project = keystone.get_project(kc, project_id)
info['project_id'] = '%s (%s)' % (project.name, project.id)
except Exception:
pass
# User
user_id = info.get('user_id')
if user_id:
try:
user = keystone.get_user(kc, user_id)
info['user_id'] = '%s (%s)' % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop('links', None)
info.pop('addresses', None)
info.pop('hostId', None)
info.pop('security_groups', None)
return _format_instance(info, style=style)
def all_servers(client, zone=None, host=None, status=None, ip=None,
image=None, project=None, user=None, limit=None,
changes_since=None):
print("\nListing the instances... ", end="")
marker = None
opts = {}
opts["all_tenants"] = True
if status:
opts['status'] = status
if limit:
opts['limit'] = limit
if image:
opts['image'] = image
if changes_since:
opts['changes-since'] = changes_since
if project:
try:
opts['tenant_id'] = keystone.get_project(
keystone.client(), project, use_cache=True).id
except Exception:
sys.exit(1)
if user:
try:
opts['user_id'] = keystone.get_user(
keystone.client(), user, use_cache=True).id
except Exception:
sys.exit(1)
host_list = parse_nodes(host) if host else None
az_list = parse_nodes(zone) if zone else None
ip_list = parse_nodes(ip) if ip else None
# When using all the searching opts other than project or user,
# trove instances will be returned by default via nova list api.
# But they will not when search_opts contain project or user.
# In order to include them, searching all the instances under
# project "trove" and filtering them by the instance metadata.
if project or user:
inst = _search_trove_instances(client, opts)
else:
inst = []
if host_list:
for host in host_list:
opts['host'] = host
instances = client.servers.list(search_opts=opts)
if not instances:
continue
instances = filter(lambda x: _match_availability_zone(x, az_list),
instances)
instances = filter(lambda x: _match_ip_address(x, ip_list),
instances)
inst.extend(instances)
if limit and len(inst) >= int(limit):
break
return inst
else:
while True:
if marker:
opts['marker'] = marker
instances = client.servers.list(search_opts=opts)
if not instances:
return inst
# for some instances stuck in build phase, servers.list api
# will always return the marker instance. Add old marker and
# new marker comparision to avoid the dead loop
marker_new = instances[-1].id
if marker == marker_new:
return inst
marker = marker_new
instances = filter(lambda x: _match_availability_zone(x, az_list),
instances)
instances = filter(lambda x: _match_ip_address(x, ip_list),
instances)
if not instances:
continue
inst.extend(instances)
if limit and len(inst) >= int(limit):
return inst