def get_instance_usage_csv(start_date=None, end_date=None, filename=None):
"""Get instance usage statistics for all projects.
Date strings should be ISO 8601 to minute precision
without timezone information.
"""
assert start_date and end_date
start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
keystone = hm_keystone.client()
nova = hm_nova.client()
tenants = {x.id: x for x in keystone.tenants.list()}
headings = ["Tenant ID", "Tenant Name", "Instance count",
"Instance hours", "vCPU hours", "Memory Hours (MB)",
"Disk hours (GB)"]
usage = map(lambda u: [
u.tenant_id,
tenants[u.tenant_id].name if u.tenant_id in tenants else None,
len(u.server_usages),
u.total_hours,
u.total_vcpus_usage,
u.total_memory_mb_usage,
u.total_local_gb_usage],
nova.usage.list(start, end, detailed=True))
csv_output(headings, usage, filename=filename)
def archive(image_id, dry_run=True, tenant=None, community=False):
"""Archive image by moving it to the <NECTAR_ARCHIVES> tenant.
If the community flag is set
please specify the community archive tenant id.
"""
if dry_run:
print("Running in dry run mode")
if community:
archive_tenant = get_community_archive_tenant(tenant)
else:
archive_tenant = get_archive_tenant(tenant)
gc = get_glance_client(keystone.client())
try:
image = gc.images.get(image_id)
except exc.HTTPNotFound:
error("Image ID not found.")
if dry_run:
print("Would archive image {} ({}) to tenant {} ({})"
.format(image.name, image.id,
archive_tenant.name, archive_tenant.id))
if 'murano_image_info' in image.properties:
print('Would remove murano image properties from {}'
.format(image.id))
else:
print("Archiving image {} ({}) to tenant {} ({})"
.format(image.name, image.id,
archive_tenant.name, archive_tenant.id))
change_tenant(image, archive_tenant)
if 'murano_image_info' in image.properties:
print('Removing murano image properties from {}'.format(image.id))
remove_property(image, 'murano_image_info')
def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.get_glance_client(kc)
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info["%s network" % network_label] = ", ".join(address_list)
flavor = info.get("flavor", {})
flavor_id = flavor.get("id", "")
try:
info["flavor"] = "%s (%s)" % (nova.get_flavor(nc, flavor_id).name, flavor_id)
except Exception:
info["flavor"] = "%s (%s)" % ("Flavor not found", flavor_id)
# Image
image = info.get("image", {})
if image:
image_id = image.get("id", "")
try:
img = gc.images.get(image_id)
nectar_build = img.properties.get("nectar_build", "N/A")
info["image"] = "%s (%s, NeCTAR Build %s)" % (img.name, img.id, nectar_build)
except Exception:
info["image"] = "Image not found (%s)" % image_id
else: # Booted from volume
info["image"] = "Attempt to boot from volume - no image supplied"
# Tenant
tenant_id = info.get("tenant_id")
if tenant_id:
try:
tenant = keystone.get_tenant(kc, tenant_id)
info["tenant_id"] = "%s (%s)" % (tenant.name, tenant.id)
except Exception:
pass
# User
user_id = info.get("user_id")
if user_id:
try:
user = keystone.get_user(kc, user_id)
info["user_id"] = "%s (%s)" % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop("links", None)
info.pop("addresses", None)
info.pop("hostId", None)
info.pop("security_groups", None)
return _format_instance(info, style=style)
def allocation_homes(csv=False, filename=None, sslwarnings=False):
"""Get the allocation_homes for all projects. If this
metadata field is not set in keystone (see keystone hivemind
commands), the value reported is the email domains for all
tenant managers belonging to this project.
"""
ssl_warnings(enabled=sslwarnings)
keystone = hm_keystone.client()
all_users = map(lambda x: x.to_dict(), keystone.users.list())
email_dict = {x['id']: x['email'].split("@")[-1] for x in all_users
if 'email' in x and x['email'] is not None}
projects = keystone.projects.list()
managers = collections.defaultdict(list)
for user_role in keystone.role_assignments.list(role=14):
if 'project' in user_role.scope:
managers[user_role.scope['project']['id']].append(
user_role.user['id'])
headings = ["Tenant ID", "Allocation Home(s)"]
records = []
for proj in projects:
if "allocation_home" in proj.to_dict():
records.append([proj.id, proj.allocation_home])
else:
if len(managers[proj.id]) == 0:
continue
institutions = set()
for tm in managers[proj.id]:
if tm in email_dict:
institutions.add(email_dict[tm])
records.append([proj.id, ",".join(institutions)])
if csv:
csv_output(headings, records, filename=filename)
else:
pretty_output(headings, records, filename=filename)
def allocation_managers(csv=False, filename=None, sslwarnings=False):
"""Get the allocation manager emails for all projects.
"""
ssl_warnings(enabled=sslwarnings)
keystone = hm_keystone.client()
all_users = map(lambda x: x.to_dict(), keystone.users.list())
email_dict = {x['id']: x['email'] for x in all_users
if 'email' in x and x['email'] is not None}
projects = keystone.projects.list()
managers = collections.defaultdict(list)
for user_role in keystone.role_assignments.list(role=14):
if 'project' in user_role.scope:
managers[user_role.scope['project']['id']].append(
user_role.user['id'])
headings = ["Tenant ID", "Manager email(s)"]
records = []
for proj in projects:
if len(managers[proj.id]) == 0:
continue
emails = set()
for tm in managers[proj.id]:
if tm in email_dict:
emails.add(email_dict[tm])
records.append([proj.id, ",".join(emails)])
if csv:
csv_output(headings, records, filename=filename)
else:
pretty_output(headings, records, filename=filename)
def promote(image_id, dry_run=True, tenant=None, community=False):
"""If the supplied image has nectar_name and nectar_build metadata, set
to public. If there is an image with matching nectar_name and lower
nectar_build, move that image to the <NECTAR_ARCHIVES> tenant.
If the community flag is set please specify the community tenant id.
"""
if dry_run:
print("Running in dry run mode")
if community:
archive_tenant = get_community_tenant(tenant)
else:
archive_tenant = get_archive_tenant(tenant)
images = get_glance_client(keystone.client()).images
try:
image = images.get(image_id)
except exc.HTTPNotFound:
error("Image ID not found.")
if not community:
try:
name = image.properties['nectar_name']
build = (int(image.properties['nectar_build']))
except KeyError:
error("nectar_name or nectar_build not found for image.")
m_check = partial(match, name, build)
matchingimages = filter(m_check, images.findall(owner=image.owner))
else:
matchingimages = [image]
for i in matchingimages:
if dry_run:
print("Would change ownership of image {} ({}) to tenant {} ({})"
.format(i.name, i.id,
archive_tenant.name, archive_tenant.id))
if 'murano_image_info' in i.properties:
print('Would remove murano image properties from {}'
.format(i.id))
else:
change_tenant(i, archive_tenant)
print("Changing ownership of image {} ({}) to tenant {} ({})"
.format(i.name, i.id,
archive_tenant.name, archive_tenant.id))
if 'murano_image_info' in i.properties:
print('Removing murano image properties from {}'
.format(i.id))
remove_property(i, 'murano_image_info')
if image.is_public:
print("Image {} ({}) already set public"
.format(image.name, image.id))
else:
if dry_run:
print("Would set image {} ({}) to public"
.format(image.name, image.id))
else:
print("Setting image {} ({}) to public"
.format(image.name, image.id))
image.update(is_public=True)
def public_audit():
"""Print usage information about all public images
"""
gc = get_glance_client(keystone.client(), api_version=2)
nc = nova.client()
db = nova.db_connect()
# The visibility filter doesn't seem to work... so we filter them out again
images = gc.images.list(visibility='public')
public = [i for i in images if i['visibility'] == 'public']
table = PrettyTable(["ID", "Name", "Num running instances",
"Boot count", "Last Boot"])
for i in public:
sql = select([nova.instances_table])
where = [nova.instances_table.c.image_ref.like(i['id'])]
sql = sql.where(*where).order_by(desc('created_at'))
image_instances = db.execute(sql).fetchall()
boot_count = len(image_instances)
if boot_count > 0:
last_boot = image_instances[0].created_at
else:
last_boot = 'Never'
instances = nova.all_servers(nc, image=i['id'])
table.add_row([i['id'], i['name'],
len(instances), boot_count, last_boot])
print(table.get_string(sortby="Num running instances", reversesort=True))
def revoke_access(user):
"""Revoke access to reports.rc.nectar.org.au for the user.
"""
ksclient = keystone.client()
user = keystone.get_user(ksclient, user)
keystone.remove_project_roles(user.default_project_id, user.id, [ROLE])
keystone.print_members(ksclient, user.default_project_id)
def grant_access(user):
"""Grant the user access to reports.rc.nectar.org.au.
"""
ksclient = keystone.client()
user = keystone.get_user(ksclient, user)
keystone.add_project_roles(user.default_project_id, user.id, [ROLE])
keystone.print_members(ksclient, user.default_project_id)
def _populate_project_dict(instances):
session = keystone.get_session()
ksclient = keystone.client(session=session)
project = collections.defaultdict(dict)
for instance in instances:
if instance['project_name'] in project.keys():
project[instance['project_name']]['instances'].append(instance)
else:
cclist = []
for role in ['TenantManager', 'Member']:
members = keystone.list_members(instance['project'], role)
if not members:
continue
for uid in members.keys():
user = keystone.get_user(ksclient, uid, use_cache=True)
if getattr(user, 'enabled', None) and \
getattr(user, 'email', None):
cclist.append(user.email)
# rule out the projects where has no valid recipients(tempest. etc)
if cclist:
project[instance['project_name']] = {'instances': [instance]}
project[instance['project_name']].update(
{'recipients': cclist})
return project
def link_account(existing_email, new_email):
db = connect()
ids = keystone_ids_from_email(db, existing_email)
new_email_ids = keystone_ids_from_email(db, new_email)
if len(ids) != 1:
print('User has multiple accounts with email %s' % existing_email)
return
user_id = ids[0]
orphan_user_id = new_email_ids[0]
print('%s: %s' % (existing_email, user_id))
print('%s: %s' % (new_email, orphan_user_id))
if user_id == orphan_user_id:
print('Those accounts are already linked')
return
client = keystone.client()
user = client.users.get(orphan_user_id)
project = client.projects.get(user.default_project_id)
servers = nova.client().servers.list(search_opts={
'all_tenants': True, 'project_id': project.id})
if len(servers):
print('Soon to be orphaned project has active instances.')
print('Advise user to terminate them.')
return
print()
print('Confirm that you want to:')
print(' - Link %s to account %s' % (new_email, existing_email))
print(' - Disable orphan Keystone project %s' % (project.name))
print(' - Disable orphan Keystone user %s' % (user.name))
print()
response = raw_input('(yes/no): ')
if response != 'yes':
return
print('Linking account.')
sql = (update(users)
.where(users.c.email == new_email)
.values(user_id=user_id))
result = db.execute(sql)
if result.rowcount == 0:
print('Something went wrong.')
return
print('Disabling orphaned Keystone project %s (%s).' % (
project.name, project.id))
client.projects.update(project.id, enabled=False)
print('Disabling orphaned Keystone user %s (%s).' % (user.name, user.id))
client.users.update(user.id, enabled=False, name="%s-disabled" % user.name)
print('All done.')
def link_account(existing_email, new_email):
db = connect()
ids = keystone_ids_from_email(db, existing_email)
new_email_ids = keystone_ids_from_email(db, new_email)
if len(ids) != 1:
print 'User has multiple accounts with email %s' % existing_email
return
user_id = ids[0]
orphan_user_id = new_email_ids[0]
print '%s: %s' % (existing_email, user_id)
print '%s: %s' % (new_email, orphan_user_id)
if user_id == orphan_user_id:
print 'Those accounts are already linked'
return
client = keystone.client()
user = client.users.get(orphan_user_id)
project = client.tenants.get(user.tenantId)
servers = nova.client().servers.list(search_opts={
'all_tenants': True, 'project_id': project.id})
if len(servers):
print 'Soon to be orphaned project has active instances.'
print 'Advise user to terminate them.'
return
print
print 'Confirm that you want to:'
print ' - Link %s to account %s' % (new_email, existing_email)
print ' - Delete orphan Keystone project %s' % (project.name)
print ' - Delete orphan Keystone user %s' % (user.name)
print
response = raw_input('(yes/no): ')
if response != 'yes':
return
print 'Linking account.'
sql = (update(users)
.where(users.c.email == new_email)
.values(user_id=user_id))
result = db.execute(sql)
if result.rowcount == 0:
print 'Something went wrong.'
return
print 'Deleting orphaned Keystone project %s (%s).' % (
project.name, project.id)
client.tenants.delete(project.id)
print 'Deleting orphaned Keystone user %s (%s).' % (user.name, user.id)
client.users.delete(user.id)
print 'All done.'
def link_account(existing_email, new_email):
db = connect()
ids = keystone_ids_from_email(db, existing_email)
new_email_ids = keystone_ids_from_email(db, new_email)
if len(ids) != 1:
print("User has multiple accounts with email %s" % existing_email)
return
user_id = ids[0]
orphan_user_id = new_email_ids[0]
print("%s: %s" % (existing_email, user_id))
print("%s: %s" % (new_email, orphan_user_id))
if user_id == orphan_user_id:
print("Those accounts are already linked")
return
client = keystone.client()
user = client.users.get(orphan_user_id)
project = client.tenants.get(user.tenantId)
servers = nova.client().servers.list(search_opts={"all_tenants": True, "project_id": project.id})
if len(servers):
print("Soon to be orphaned project has active instances.")
print("Advise user to terminate them.")
return
print()
print("Confirm that you want to:")
print(" - Link %s to account %s" % (new_email, existing_email))
print(" - Delete orphan Keystone project %s" % (project.name))
print(" - Delete orphan Keystone user %s" % (user.name))
print()
response = raw_input("(yes/no): ")
if response != "yes":
return
print("Linking account.")
sql = update(users).where(users.c.email == new_email).values(user_id=user_id)
result = db.execute(sql)
if result.rowcount == 0:
print("Something went wrong.")
return
print("Deleting orphaned Keystone project %s (%s)." % (project.name, project.id))
client.tenants.delete(project.id)
print("Deleting orphaned Keystone user %s (%s)." % (user.name, user.id))
client.users.delete(user.id)
print("All done.")
def _search_trove_instances(client, opts):
# keep the proj/user from searching opts
proj_id = opts.get('tenant_id', None)
user_id = opts.get('user_id', None)
# trove instances will be launched by global trove project
trove_opts = opts.copy()
trove_opts.pop('user_id', None)
trove_opts['tenant_id'] = keystone.get_project(
keystone.client(), 'trove', use_cache=True).id
trove_instances = client.servers.list(search_opts=trove_opts)
trove_instances = [instance for instance in trove_instances
if _match_proj_user(instance, proj_id, user_id)]
return trove_instances
def list_instances(zone=None, nodes=None, project=None, user=None,
status="ACTIVE", ip=None, image=None, limit=None,
changes_since=None, scenario=None):
"""Prints a pretty table of instances based on specific conditions
:param str zone: Availability zone or availability zone range
that the instances are in, e.g. az[1-4,9]
:param str nodes: Compute host name or host neme range that the
instances are in, e.g. cc[2-3,5]
:param str project: Project name or id that the instances belong to
:param str user: User name or id that the instances belong to
:param str status: Instances status. Use 'ALL' to list all instances
:param str ip: Ip address or ip address range that instances are in,
e.g. 192.168.122.[124-127]
:param str image: Image id that the instances are launched based on
:param str limit: Number of returned instances
:param str changes_since: List only instances changed after a certain
point of time. The provided time should be an ISO 8061 formatted
time. e.g. 2016-03-04T06:27:59Z
:param str scenario: List only instances which match with specific
scenario checking, available ones are ["compute_failure"]
"""
novaclient = client()
if status == 'ALL':
status = None
result = all_servers(novaclient, zone=zone, host=nodes, status=status,
ip=ip, image=image, project=project, user=user,
limit=limit, changes_since=changes_since)
if not result:
print("No instances found!")
sys.exit(0)
result = extract_servers_info(result, keystone.client())
if scenario:
func = globals()["_scenario_" + scenario]
result = match_scenario(result, func, novaclient, changes_since)
if not result:
print("No %s instances found!" % scenario)
sys.exit(0)
print("\n")
header = None
for inst in result:
if not header:
header = inst.keys()
table = PrettyTable(header)
table.add_row(inst.values())
print(table)
print("number of instances:", len(result))
return result
def get_ticket_recipients(instance):
"""Build a list of email addresses"""
email_addresses = []
kc = keystone.client()
user = keystone.get_user(kc, instance.user_id)
if user.email:
email_addresses.append(user.email)
# Add tenant members to ticket recipient list
tenant = keystone.get_tenant(kc, instance.tenant_id)
for user in tenant.list_users():
roles = [r.name for r in user.list_roles(tenant)]
if 'TenantManager' in roles:
email_addresses.append(user.email)
return email_addresses
def purge_project(tenant_name, dry_run=True):
"""Purge resources and disable a given project """
if not spawn.find_executable('ospurge'):
error('ospurge not found in path. Please ensure it is installed.')
username, password = get_creds()
if dry_run:
print("Running in dry-run mode")
ks_client = keystone.client()
tenant = keystone.get_tenant(ks_client, tenant_name)
tenant_member_role = ks_client.roles.find(name='Member')
ospurge_user = keystone.get_user(ks_client, username)
if not tenant.enabled:
print("Enabling project for purge")
tenant.update(enabled=True)
if not is_in_project(tenant, ospurge_user):
print("Adding ospurge user to project")
tenant.add_user(ospurge_user, tenant_member_role)
if dry_run:
run_opt = '--dry-run'
else:
run_opt = '--verbose'
cmd = ("ospurge --dont-delete-project --own-project --username {username} "
"--password {password} --admin-project {tenant} {run_opt}"
"".format(username=username, password=password,
tenant=tenant.name, run_opt=run_opt))
print("Running: {}".format(cmd.replace(password, 'xxxx')))
run(cmd)
if is_in_project(tenant, ospurge_user):
print("Removing ospurge user from project")
tenant.remove_user(ospurge_user, tenant_member_role)
if tenant.enabled:
if dry_run and tenant.enabled:
print("Not disabling project due to dry-run")
else:
print("Disabling project")
tenant.update(enabled=False)
keystone.set_project_metadata(tenant_name,
'ospurge_date',
str(datetime.datetime.now()))
def get_images_tenant(tenant_id_or_name, tenant_type):
"""fetch tenant id from config file"""
if tenant_id_or_name is None:
msg = " ".join(("No tenant set.", "Please set tenant in",
"[cfg:hivemind_contrib.glance.%s]" % tenant_type))
error(msg)
try:
ks_client = keystone.client()
tenant = keystone.get_tenant(ks_client, tenant_id_or_name)
except ks_exc.NotFound:
raise error("Tenant {} not found. Check your settings."
.format(tenant_id_or_name))
except ks_exc.Forbidden:
raise error("Permission denied. Check you're using admin credentials.")
except Exception as e:
raise error(e)
return tenant
def create_zone(zone_name, project_id_or_name, dry_run=True):
"""Create a designate zone for a user"""
# Designate requires zone names to end with a dot.
if not zone_name.endswith('.'):
zone_name = "%s." % zone_name
d_client = client()
try:
ks_client = keystone.client()
project = keystone.get_project(ks_client, project_id_or_name)
except ks_exc.NotFound:
raise error("Project {} not found. Check your settings."
.format(project_id_or_name))
except ks_exc.Forbidden:
raise error("Permission denied getting project {} details."
.format(project_id_or_name))
if dry_run:
print("Would create designate zone {}".format(zone_name))
print("Would transfer zone {} to project {}".format(
zone_name, project.id))
else:
d_client.session.sudo_project_id = None
zone = d_client.zones.create(zone_name,
email='*****@*****.**')
print("Created new zone {}".format(zone['name']))
print("Transferring zone {} to project {}".format(
zone['name'], project.id))
create_req = d_client.zone_transfers.create_request(
zone_name, project.id)
d_client.session.sudo_project_id = project.id
accept_req = d_client.zone_transfers.accept_request(
create_req['id'], create_req['key'])
if accept_req['status'] == 'COMPLETE':
print("Zone {} transfer to project {} is complete".format(
zone['name'], project.id))
return zone
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd = get_freshdesk_client()
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
tenant = keystone.get_tenant(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email_addresses = get_ticket_recipients(instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {}'.format(instance_id)
body = '<br />\n'.join([
'Dear NeCTAR Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance.id),
'in the project <b>{}</b>'.format(tenant.name),
'created by <b>{}</b>'.format(user.email),
'has been involved in a security incident.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'NeCTAR Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {}'.format(email_addresses))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_ticket(
description=body,
subject=subject,
email='*****@*****.**',
cc_emails=email_addresses,
priority=4,
status=6,
tags=['security'])
ticket_id = ticket.id
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(fd.domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
print('Would update ticket with action')
else:
# Pause and lock
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Add reply to user
email_addresses = get_ticket_recipients(instance)
print('Replying to ticket with action details')
action = 'Instance <b>{} ({})</b> has been <b>paused and locked</b> '\
'pending further investigation'\
.format(instance.name, instance_id)
fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
def all_servers(client, zone=None, host=None, status=None, ip=None,
image=None, project=None, user=None, limit=None,
changes_since=None):
print("\nListing the instances... ", end="")
marker = None
opts = {}
opts["all_tenants"] = True
if status:
opts['status'] = status
if limit:
opts['limit'] = limit
if image:
opts['image'] = image
if changes_since:
opts['changes-since'] = changes_since
if project:
try:
opts['tenant_id'] = keystone.get_project(
keystone.client(), project, use_cache=True).id
except Exception:
sys.exit(1)
if user:
try:
opts['user_id'] = keystone.get_user(
keystone.client(), user, use_cache=True).id
except Exception:
sys.exit(1)
host_list = parse_nodes(host) if host else None
az_list = parse_nodes(zone) if zone else None
ip_list = parse_nodes(ip) if ip else None
# When using all the searching opts other than project or user,
# trove instances will be returned by default via nova list api.
# But they will not when search_opts contain project or user.
# In order to include them, searching all the instances under
# project "trove" and filtering them by the instance metadata.
if project or user:
inst = _search_trove_instances(client, opts)
else:
inst = []
if host_list:
for host in host_list:
opts['host'] = host
instances = client.servers.list(search_opts=opts)
if not instances:
continue
instances = filter(lambda x: _match_availability_zone(x, az_list),
instances)
instances = filter(lambda x: _match_ip_address(x, ip_list),
instances)
inst.extend(instances)
if limit and len(inst) >= int(limit):
break
return inst
else:
while True:
if marker:
opts['marker'] = marker
instances = client.servers.list(search_opts=opts)
if not instances:
return inst
# for some instances stuck in build phase, servers.list api
# will always return the marker instance. Add old marker and
# new marker comparision to avoid the dead loop
marker_new = instances[-1].id
if marker == marker_new:
return inst
marker = marker_new
instances = filter(lambda x: _match_availability_zone(x, az_list),
instances)
instances = filter(lambda x: _match_ip_address(x, ip_list),
instances)
if not instances:
continue
inst.extend(instances)
if limit and len(inst) >= int(limit):
return inst
def extract_servers_info(servers, ksclient=None):
print("\nExtracting instances information... ", end="")
if ksclient is None:
ksclient = keystone.client()
return [extract_server_info(server, ksclient=ksclient)
for server in servers]
def get_instance_usage_csv(start_date, end_date, tenant=None,
filename=None, sslwarnings=False):
"""Get individual instance usage for all projects, including tenant and
availability zones. Date strings should be ISO 8601 to minute precision
without timezone information.
"""
ssl_warnings(enabled=sslwarnings)
start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
keystone = hm_keystone.client()
nova = hm_nova.client()
if tenant:
tenants = {tenant: keystone.projects.get(tenant)}
raw_usage = [nova.usage.get(tenant, start, end)]
else:
tenants = {x.id: x for x in keystone.projects.list()}
raw_usage = nova.usage.list(start, end, detailed=True)
usage = []
for u in raw_usage:
tenant_id = u.tenant_id
tenant_name = tenants[tenant_id].name if tenant_id in tenants else None
# The Nova API doesn't allow "show" on deleted instances, but
# we can get the info using "list --deleted". The problem is
# figuring out how to avoid retrieving irrelevant instances,
# and at the same time how to avoid too many requests.
#
# Attempt #1 - use the tenant_id and the instance's name to
# focus queries.
# Attempt #2 - as #1, but after N lookups by name for a tenant,
# just fetch all of the deleted instances.
cache = {}
try:
for iu in u.server_usages:
name = iu['name']
instance_id = iu['instance_id']
instance = None
if iu['state'] == 'terminated' or iu['state'] == 'deleted':
instance = _get_deleted_instance(cache, nova, u.tenant_id,
name, instance_id)
else:
try:
instance = nova.servers.get(instance_id).to_dict()
except Exception:
print('Cannot find instance {0} in {1}'
.format(instance_id, u.tenant_id))
if instance is None:
instance = {'OS-EXT-AZ:availability_zone': 'unknown'}
usage.append([tenant_id, tenant_name, instance_id, name,
iu['state'], iu['flavor'], iu['hours'],
iu['vcpus'], iu['memory_mb'], iu['local_gb'],
instance['OS-EXT-AZ:availability_zone']])
except Exception:
traceback.print_exc(file=sys.stdout)
headings = ["Tenant ID", "Tenant Name", "Instance id", "Instance name",
"Instance state", "Flavour",
"Instance hours", "vCPUs", "Memory (MB)", "Disk (GB)", "AZ"]
csv_output(headings, usage, filename=filename)
def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info['%s network' % network_label] = ', '.join(address_list)
flavor = info.get('flavor', {})
flavor_id = flavor.get('id', '')
try:
info['flavor'] = '%s (%s)' % (nova.get_flavor(nc, flavor_id).name,
flavor_id)
except Exception:
info['flavor'] = '%s (%s)' % ("Flavor not found", flavor_id)
# Image
image = info.get('image', {})
if image:
image_id = image.get('id', '')
try:
img = gc.images.get(image_id)
nectar_build = img.get('nectar_build', 'N/A')
info['image'] = ('%s (%s, NeCTAR Build %s)'
% (img.name, img.id, nectar_build))
except Exception:
info['image'] = 'Image not found (%s)' % image_id
else: # Booted from volume
info['image'] = "Attempt to boot from volume - no image supplied"
# Tenant
project_id = info.get('tenant_id')
if project_id:
try:
project = keystone.get_project(kc, project_id)
info['project_id'] = '%s (%s)' % (project.name, project.id)
except Exception:
pass
# User
user_id = info.get('user_id')
if user_id:
try:
user = keystone.get_user(kc, user_id)
info['user_id'] = '%s (%s)' % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop('links', None)
info.pop('addresses', None)
info.pop('hostId', None)
info.pop('security_groups', None)
return _format_instance(info, style=style)
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd_config = get_freshdesk_config()
fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key'])
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
# Pause and lock instance
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
else:
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Process ticket
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
project = keystone.get_project(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email = user.email or '*****@*****.**'
name = getattr(user, 'full_name', email)
cc_emails = get_tenant_managers_emails(kc, instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {} ({})'.format(
instance.name, instance_id)
body = '<br />\n'.join([
'Dear Nectar Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance_id),
'in the project <b>{}</b>'.format(project.name),
'created by <b>{}</b>'.format(email),
'has been involved in a security incident, and has been locked.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'Nectar Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {} <{}>'.format(name, email))
print(' CC: {}'.format(', '.join(cc_emails)))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_outbound_email(
name=name,
description=body,
subject=subject,
email=email,
cc_emails=cc_emails,
email_config_id=int(fd_config['email_config_id']),
group_id=int(fd_config['group_id']),
priority=4,
status=2,
tags=['security'])
ticket_id = ticket.id
# Use friendly domain name if using prod
if fd.domain == 'dhdnectar.freshdesk.com':
domain = 'support.ehelp.edu.au'
else:
domain = fd.domain
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)