def get_project_usage_csv(start_date=None, end_date=None,
filename=None, sslwarnings=False):
"""Get accumulated instance usage statistics for all projects.
Date strings should be ISO 8601 to minute precision
without timezone information.
"""
ssl_warnings(enabled=sslwarnings)
assert start_date and end_date
start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
keystone = hm_keystone.client_session(version=3)
nova = hm_nova.client()
tenants = {x.id: x for x in keystone.projects.list()}
headings = ["Tenant ID", "Tenant Name", "Instance count",
"Instance hours", "vCPU hours", "Memory Hours (MB)",
"Disk hours (GB)"]
usage = map(lambda u: [
u.tenant_id,
tenants[u.tenant_id].name if u.tenant_id in tenants else None,
len(u.server_usages),
u.total_hours,
u.total_vcpus_usage,
u.total_memory_mb_usage,
u.total_local_gb_usage],
nova.usage.list(start, end, detailed=True))
csv_output(headings, usage, filename=filename)
def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.get_glance_client(kc)
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info["%s network" % network_label] = ", ".join(address_list)
flavor = info.get("flavor", {})
flavor_id = flavor.get("id", "")
try:
info["flavor"] = "%s (%s)" % (nova.get_flavor(nc, flavor_id).name, flavor_id)
except Exception:
info["flavor"] = "%s (%s)" % ("Flavor not found", flavor_id)
# Image
image = info.get("image", {})
if image:
image_id = image.get("id", "")
try:
img = gc.images.get(image_id)
nectar_build = img.properties.get("nectar_build", "N/A")
info["image"] = "%s (%s, NeCTAR Build %s)" % (img.name, img.id, nectar_build)
except Exception:
info["image"] = "Image not found (%s)" % image_id
else: # Booted from volume
info["image"] = "Attempt to boot from volume - no image supplied"
# Tenant
tenant_id = info.get("tenant_id")
if tenant_id:
try:
tenant = keystone.get_tenant(kc, tenant_id)
info["tenant_id"] = "%s (%s)" % (tenant.name, tenant.id)
except Exception:
pass
# User
user_id = info.get("user_id")
if user_id:
try:
user = keystone.get_user(kc, user_id)
info["user_id"] = "%s (%s)" % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop("links", None)
info.pop("addresses", None)
info.pop("hostId", None)
info.pop("security_groups", None)
return _format_instance(info, style=style)
def public_audit():
"""Print usage information about all public images
"""
gc = get_glance_client(keystone.client(), api_version=2)
nc = nova.client()
db = nova.db_connect()
# The visibility filter doesn't seem to work... so we filter them out again
images = gc.images.list(visibility='public')
public = [i for i in images if i['visibility'] == 'public']
table = PrettyTable(["ID", "Name", "Num running instances",
"Boot count", "Last Boot"])
for i in public:
sql = select([nova.instances_table])
where = [nova.instances_table.c.image_ref.like(i['id'])]
sql = sql.where(*where).order_by(desc('created_at'))
image_instances = db.execute(sql).fetchall()
boot_count = len(image_instances)
if boot_count > 0:
last_boot = image_instances[0].created_at
else:
last_boot = 'Never'
instances = nova.all_servers(nc, image=i['id'])
table.add_row([i['id'], i['name'],
len(instances), boot_count, last_boot])
print(table.get_string(sortby="Num running instances", reversesort=True))
def get_instance_usage_csv(start_date=None, end_date=None, filename=None):
"""Get instance usage statistics for all projects.
Date strings should be ISO 8601 to minute precision
without timezone information.
"""
assert start_date and end_date
start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
keystone = hm_keystone.client()
nova = hm_nova.client()
tenants = {x.id: x for x in keystone.tenants.list()}
headings = ["Tenant ID", "Tenant Name", "Instance count",
"Instance hours", "vCPU hours", "Memory Hours (MB)",
"Disk hours (GB)"]
usage = map(lambda u: [
u.tenant_id,
tenants[u.tenant_id].name if u.tenant_id in tenants else None,
len(u.server_usages),
u.total_hours,
u.total_vcpus_usage,
u.total_memory_mb_usage,
u.total_local_gb_usage],
nova.usage.list(start, end, detailed=True))
csv_output(headings, usage, filename=filename)
def link_account(existing_email, new_email):
db = connect()
ids = keystone_ids_from_email(db, existing_email)
new_email_ids = keystone_ids_from_email(db, new_email)
if len(ids) != 1:
print('User has multiple accounts with email %s' % existing_email)
return
user_id = ids[0]
orphan_user_id = new_email_ids[0]
print('%s: %s' % (existing_email, user_id))
print('%s: %s' % (new_email, orphan_user_id))
if user_id == orphan_user_id:
print('Those accounts are already linked')
return
client = keystone.client()
user = client.users.get(orphan_user_id)
project = client.projects.get(user.default_project_id)
servers = nova.client().servers.list(search_opts={
'all_tenants': True, 'project_id': project.id})
if len(servers):
print('Soon to be orphaned project has active instances.')
print('Advise user to terminate them.')
return
print()
print('Confirm that you want to:')
print(' - Link %s to account %s' % (new_email, existing_email))
print(' - Disable orphan Keystone project %s' % (project.name))
print(' - Disable orphan Keystone user %s' % (user.name))
print()
response = raw_input('(yes/no): ')
if response != 'yes':
return
print('Linking account.')
sql = (update(users)
.where(users.c.email == new_email)
.values(user_id=user_id))
result = db.execute(sql)
if result.rowcount == 0:
print('Something went wrong.')
return
print('Disabling orphaned Keystone project %s (%s).' % (
project.name, project.id))
client.projects.update(project.id, enabled=False)
print('Disabling orphaned Keystone user %s (%s).' % (user.name, user.id))
client.users.update(user.id, enabled=False, name="%s-disabled" % user.name)
print('All done.')
def get_instance_usage_csv(start_date=None, end_date=None,
filename=None, sslwarnings=False):
"""Get individual instance usage for all projects, including tenant and
availability zones. Date strings should be ISO 8601 to minute precision
without timezone information.
"""
ssl_warnings(enabled=sslwarnings)
assert start_date and end_date
start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
keystone = hm_keystone.client_session(version=3)
nova = hm_nova.client()
tenants = {x.id: x for x in keystone.projects.list()}
usage = []
for u in nova.usage.list(start, end, detailed=True):
tenant_id = u.tenant_id
tenant_name = tenants[tenant_id].name if tenant_id in tenants else None
# The Nova API doesn't allow "show" on deleted instances, but
# we can get the info using "list --deleted". The problem is
# figuring out how to avoid retrieving irrelevant instances,
# and at the same time how to avoid too many requests.
#
# Attempt #1 - use the tenant_id and the instance's name to
# focus queries.
# Attempt #2 - as #1, but after N lookups by name for a tenant,
# just fetch all of the deleted instances.
cache = {}
try:
for iu in u.server_usages:
name = iu['name']
instance_id = iu['instance_id']
instance = None
if iu['state'] == 'terminated' or iu['state'] == 'deleted':
instance = _get_deleted_instance(cache, nova, u.tenant_id,
name, instance_id)
else:
try:
instance = nova.servers.get(instance_id).to_dict()
except:
print 'Cannot find instance {0} in {1}' \
.format(instance_id, u.tenant_id)
if instance is None:
instance = {'OS-EXT-AZ:availability_zone': 'unknown'}
usage.append([tenant_id, tenant_name, instance_id, name,
iu['state'], iu['flavor'], iu['hours'],
iu['vcpus'], iu['memory_mb'], iu['local_gb'],
instance['OS-EXT-AZ:availability_zone']])
except:
traceback.print_exc(file=sys.stdout)
headings = ["Tenant ID", "Tenant Name", "Instance id", "Instance name",
"Instance state", "Flavour",
"Instance hours", "vCPUs", "Memory (MB)", "Disk (GB)", "AZ"]
csv_output(headings, usage, filename=filename)
def link_account(existing_email, new_email):
db = connect()
ids = keystone_ids_from_email(db, existing_email)
new_email_ids = keystone_ids_from_email(db, new_email)
if len(ids) != 1:
print 'User has multiple accounts with email %s' % existing_email
return
user_id = ids[0]
orphan_user_id = new_email_ids[0]
print '%s: %s' % (existing_email, user_id)
print '%s: %s' % (new_email, orphan_user_id)
if user_id == orphan_user_id:
print 'Those accounts are already linked'
return
client = keystone.client()
user = client.users.get(orphan_user_id)
project = client.tenants.get(user.tenantId)
servers = nova.client().servers.list(search_opts={
'all_tenants': True, 'project_id': project.id})
if len(servers):
print 'Soon to be orphaned project has active instances.'
print 'Advise user to terminate them.'
return
print
print 'Confirm that you want to:'
print ' - Link %s to account %s' % (new_email, existing_email)
print ' - Delete orphan Keystone project %s' % (project.name)
print ' - Delete orphan Keystone user %s' % (user.name)
print
response = raw_input('(yes/no): ')
if response != 'yes':
return
print 'Linking account.'
sql = (update(users)
.where(users.c.email == new_email)
.values(user_id=user_id))
result = db.execute(sql)
if result.rowcount == 0:
print 'Something went wrong.'
return
print 'Deleting orphaned Keystone project %s (%s).' % (
project.name, project.id)
client.tenants.delete(project.id)
print 'Deleting orphaned Keystone user %s (%s).' % (user.name, user.id)
client.users.delete(user.id)
print 'All done.'
def sync_vm_rules(project_id=None):
"""Sync security groups for the given instance UUID (-I)"""
if not env.instance_uuid and project_id is None:
error("No instance ID or project specified.")
if env.instance_uuid:
uuid = env.instance_uuid
nova_client = client()
server = nova_client.servers.get(uuid)
project_id = server.tenant_id
with show('stdout', 'stderr'):
run('nova-manage project sync_secgroups %s' % project_id)
def sync_vm_rules(project_id=None):
"""Sync security groups for the given instance UUID (-I)"""
if not env.instance_uuid and project_id is None:
error("No instance ID or project specified.")
if env.instance_uuid:
uuid = env.instance_uuid
nova_client = client()
server = nova_client.servers.get(uuid)
project_id = server.tenant_id
with show('stdout', 'stderr'):
run('nova-manage project sync_secgroups %s' % project_id)
def link_account(existing_email, new_email):
db = connect()
ids = keystone_ids_from_email(db, existing_email)
new_email_ids = keystone_ids_from_email(db, new_email)
if len(ids) != 1:
print("User has multiple accounts with email %s" % existing_email)
return
user_id = ids[0]
orphan_user_id = new_email_ids[0]
print("%s: %s" % (existing_email, user_id))
print("%s: %s" % (new_email, orphan_user_id))
if user_id == orphan_user_id:
print("Those accounts are already linked")
return
client = keystone.client()
user = client.users.get(orphan_user_id)
project = client.tenants.get(user.tenantId)
servers = nova.client().servers.list(search_opts={"all_tenants": True, "project_id": project.id})
if len(servers):
print("Soon to be orphaned project has active instances.")
print("Advise user to terminate them.")
return
print()
print("Confirm that you want to:")
print(" - Link %s to account %s" % (new_email, existing_email))
print(" - Delete orphan Keystone project %s" % (project.name))
print(" - Delete orphan Keystone user %s" % (user.name))
print()
response = raw_input("(yes/no): ")
if response != "yes":
return
print("Linking account.")
sql = update(users).where(users.c.email == new_email).values(user_id=user_id)
result = db.execute(sql)
if result.rowcount == 0:
print("Something went wrong.")
return
print("Deleting orphaned Keystone project %s (%s)." % (project.name, project.id))
client.tenants.delete(project.id)
print("Deleting orphaned Keystone user %s (%s)." % (user.name, user.id))
client.users.delete(user.id)
print("All done.")
def compare_quotas(name_or_id=None):
"""Compare the allocation and quota information for a tenant
"""
if name_or_id == None:
print 'A tenant name or id is required'
return
keystone_api = hm_keystone.client_session(version=3)
try:
tenant = hm_keystone.get_tenant(keystone_api, name_or_id)
except:
print 'Tenant {0} not found in keystone'.format(name_or_id)
return
nova_api = hm_nova.client()
quotas = nova_api.quotas.get(tenant.id)
print 'nova quotas: instances {0}, cores {1}, ram {2}'.format(
quotas.instances, quotas.cores, quotas.ram / 1024)
usage = _get_usage(nova_api, _get_flavor_map(nova_api), tenant.id)
print 'nova usage: instances {0}, cores {1}, ram {2}'.format(
usage['instances'], usage['vcpus'], usage['ram'] / 1024)
allocations_api = NectarApiSession()
allocations = allocations_api.get_allocations();
tenant_allocations = filter(lambda x: x['tenant_uuid'] == tenant.id and \
(x['status'] == 'A' or x['status'] == 'X'),
allocations)
if len(tenant_allocations) == 0:
print 'No approved allocation records for tenant {0} / {1}'.format(
tenant.id, tenant.name)
return
tenant_allocations.sort(key=lambda alloc: alloc['modified_time'])
current_allocation = tenant_allocations[-1]
format = '{0} mismatch: allocated {1}, nova {2}, used {3}'
if current_allocation['instance_quota'] != quotas.instances:
print format.format('Instance quota',
current_allocation['instance_quota'],
quotas.instances,
usage['instances'])
if current_allocation['core_quota'] != quotas.cores:
print format.format('VCPU quota',
current_allocation['core_quota'],
quotas.cores,
usage['vcpus'])
if current_allocation['ram_quota'] * 1024 != quotas.ram:
print format.format('RAM quota',
current_allocation['ram_quota'] * 1024,
quotas.ram,
usage['ram'])
def unlock_instance(instance_id, dry_run=True):
"""unlock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd = get_freshdesk_client()
nc = nova.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
else:
if not dry_run:
error('No ticket found in instance metadata!')
if dry_run:
print('Would unpause and unlock instance {}'.format(instance_id))
print('Would reply to ticket')
print('Would resolve ticket')
else:
if instance.status != 'PAUSED':
print('Instance not paused')
else:
print('Unpausing instance {}'.format(instance_id))
instance.unpause()
print('Unlocking instance {}'.format(instance_id))
instance.unlock()
# Add reply to user
email_addresses = get_ticket_recipients(instance)
print('Replying to ticket with action details')
action = 'Instance <b>{} ({})</b> has been <b>unpaused and '\
'unlocked</b>'.format(instance.name, instance_id)
fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
# Set ticket status=resolved
print('Setting ticket #{} status to resolved'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=4)
def public_audit():
"""Print usage information about all public images
"""
gc = client()
nc = nova.client()
db = nova.db_connect()
# The visibility filter doesn't seem to work... so we filter them out again
images = gc.images.list(visibility='public')
public = [i for i in images if i['visibility'] == 'public']
table = PrettyTable(["ID", "Name", "Official", "Build", "Running",
"Boots", "Last Boot"])
table.align = 'l'
table.align['Running'] = 'r'
table.align['Boots'] = 'r'
for i in public:
sql = select([nova.instances_table])
where = [nova.instances_table.c.image_ref.like(i.id)]
sql = sql.where(*where).order_by(desc('created_at'))
image_instances = db.execute(sql).fetchall()
boot_count = len(image_instances)
last_boot = 'Never'
if boot_count > 0:
last_boot = image_instances[0].created_at
instances = nova.all_servers(nc, image=i['id'])
# NeCTAR-Images, NeCTAR-Images-Archive
official_projects = ['28eadf5ad64b42a4929b2fb7df99275c',
'c9217cb583f24c7f96567a4d6530e405']
if i.owner in official_projects:
official = 'Y'
else:
official = 'N'
name = i.get('name', 'n/a')
build = i.get('nectar_build', 'n/a')
num = len(instances) if instances else 0
table.add_row([i.id, name, official, build,
num, boot_count, last_boot])
print(table.get_string(sortby="Running", reversesort=True))
def official_audit():
"""Print usage information about official images
"""
data = {}
gc = client()
nc = nova.client()
db = nova.db_connect()
images = []
# NeCTAR-Images, NeCTAR-Images-Archive
official_projects = ['28eadf5ad64b42a4929b2fb7df99275c',
'c9217cb583f24c7f96567a4d6530e405']
for project in official_projects:
images += list(gc.images.list(filters={'owner': project}))
table = PrettyTable(["Name", "Running", "Boots"])
table.align = 'l'
table.align['Running'] = 'r'
table.align['Boots'] = 'r'
for i in images:
sql = select([nova.instances_table])
where = [nova.instances_table.c.image_ref.like(i.id)]
sql = sql.where(*where).order_by(desc('created_at'))
image_instances = db.execute(sql).fetchall()
boot_count = len(image_instances)
instances = nova.all_servers(nc, image=i['id'])
if i.owner in official_projects or not i.owner:
if i.name in data:
data[i.name]['running'] += len(instances)
data[i.name]['boots'] += boot_count
else:
data[i.name] = {'running': len(instances),
'boots': boot_count}
for d in data.iteritems():
table.add_row([d[0], d[1]['running'], d[1]['boots']])
print(table.get_string(sortby="Running", reversesort=True))
def vm_rules():
if not env.instance_uuid:
error("No instance_uuid specified.")
uuid = env.instance_uuid
nova_client = client()
server = nova_client.servers.get(uuid)
host = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname')
libvirt_server = [s for s in chain(*execute(list_instances,
host=host).values())
if s['uuid'] == uuid][0]
for vm, rules in chain.from_iterable(map(dict.items,
execute(parse_rules,
host=host).values())):
if vm != str(libvirt_server['nova_id']):
continue
table = PrettyTable(['Target', 'Protocol', 'Source',
'Destination', 'Filter'])
for rule in rules:
table.add_row([rule['target'], rule['protocol'], rule['source'],
rule['destination'], rule['filter']])
puts("\n%s\n%s\n" % (server.id, str(table)))
def crosscheck_quotas(filename=None):
"""Cross-check allocation and quota information for all tenants
"""
allocations = _get_current_allocations()
nova_api = hm_nova.client()
missing = []
mismatches = []
for uuid in allocations.keys():
alloc = allocations[uuid]
try:
quotas = nova_api.quotas.get(uuid)
except:
missing.append(alloc)
continue
if (quotas.instances != alloc['instance_quota'] \
or quotas.ram != alloc['ram_quota'] * 1024 \
or quotas.cores != alloc['core_quota']):
alloc['nova_quotas'] = quotas
mismatches.append(alloc)
print '{0} allocations, {1} missing tenants, {2} quota mismatches'.format(
len(allocations), len(missing), len(mismatches))
fields_to_report = [
("Tenant ID", lambda x: x['tenant_uuid']),
("Tenant Name", lambda x: x['tenant_name']),
("Modified time", lambda x: x['modified_time']),
("Instances", lambda x: x['instance_quota']),
("Nova instances", lambda x: x['nova_quotas'].instances),
("vCPU quota", lambda x: x['core_quota']),
("Nova vCPU quota", lambda x: x['nova_quotas'].cores),
("RAM quota", lambda x: x['ram_quota'] * 1024),
("Nova RAM quota", lambda x: x['nova_quotas'].ram)
]
csv_output(map(lambda x: x[0], fields_to_report),
map(lambda alloc: map(
lambda y: y[1](alloc),
fields_to_report),
mismatches),
filename=filename)
def delete_instance(instance_id, dry_run=True):
"""delete an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd_config = get_freshdesk_config()
fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key'])
nc = nova.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
else:
if not dry_run:
error('No ticket found in instance metadata!')
# DELETE!!!
if dry_run:
print('Would delete instance {}'.format(instance_id))
print('Would reply to ticket')
print('Would resolve ticket')
else:
print('Deleting instance {})'.format(instance_id))
instance.delete()
# Add reply to user
print('Updating ticket with action')
action = 'Instance <b>{} ({})</b> has been <b>deleted.</b>'\
.format(instance.name, instance_id)
fd.comments.create_reply(ticket_id, action)
# Set ticket status=resolved
print('Resolving ticket #{}'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=4)
def crosscheck_usage(filename=None):
"""Cross-check that allocation and instantaneous usage information
for all tenants
"""
allocations = _get_current_allocations()
nova_api = hm_nova.client()
flavors = _get_flavor_map(nova_api)
missing = []
mismatches = []
for uuid in allocations.keys():
alloc = allocations[uuid]
usage = _get_usage(nova_api, flavors, uuid)
if (usage['instances'] > alloc['instance_quota']
or usage['vcpus'] > alloc['core_quota']
or usage['ram'] > alloc['ram_quota'] * 1024):
alloc['nova_usage'] = usage
mismatches.append(alloc)
print '{0} allocations, {1} missing tenants, {2} usage mismatches'.format(
len(allocations), len(missing), len(mismatches))
fields_to_report = [
("Tenant ID", lambda x: x['tenant_uuid']),
("Tenant Name", lambda x: x['tenant_name']),
("Modified time", lambda x: x['modified_time']),
("Instances", lambda x: x['instance_quota']),
("Nova instances", lambda x: x['nova_usage']['instances']),
("vCPU quota", lambda x: x['core_quota']),
("Nova vCPU usage", lambda x: x['nova_usage']['vcpus']),
("RAM quota", lambda x: x['ram_quota'] * 1024),
("Nova RAM usage", lambda x: x['nova_usage']['ram'])
]
csv_output(map(lambda x: x[0], fields_to_report),
map(lambda alloc: map(
lambda y: y[1](alloc),
fields_to_report),
mismatches),
filename=filename)
def vm_rules():
if not env.instance_uuid:
error("No instance_uuid specified.")
uuid = env.instance_uuid
nova_client = client()
server = nova_client.servers.get(uuid)
host = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname')
libvirt_server = [
s for s in chain(*execute(list_instances, host=host).values())
if s['uuid'] == uuid
][0]
for vm, rules in chain.from_iterable(
map(dict.items,
execute(parse_rules, host=host).values())):
if vm != str(libvirt_server['nova_id']):
continue
table = PrettyTable(
['Target', 'Protocol', 'Source', 'Destination', 'Filter'])
for rule in rules:
table.add_row([
rule['target'], rule['protocol'], rule['source'],
rule['destination'], rule['filter']
])
puts("\n%s\n%s\n" % (server.id, str(table)))
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd = get_freshdesk_client()
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
tenant = keystone.get_tenant(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email_addresses = get_ticket_recipients(instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {}'.format(instance_id)
body = '<br />\n'.join([
'Dear NeCTAR Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance.id),
'in the project <b>{}</b>'.format(tenant.name),
'created by <b>{}</b>'.format(user.email),
'has been involved in a security incident.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'NeCTAR Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {}'.format(email_addresses))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_ticket(
description=body,
subject=subject,
email='*****@*****.**',
cc_emails=email_addresses,
priority=4,
status=6,
tags=['security'])
ticket_id = ticket.id
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(fd.domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
print('Would update ticket with action')
else:
# Pause and lock
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Add reply to user
email_addresses = get_ticket_recipients(instance)
print('Replying to ticket with action details')
action = 'Instance <b>{} ({})</b> has been <b>paused and locked</b> '\
'pending further investigation'\
.format(instance.name, instance_id)
fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
def announcement_mailout(template, zone=None, ip=None, nodes=None, image=None,
status="ALL", project=None, user=None,
subject="Important announcement concerning your "
"instance(s)", start_time=None, duration=0,
timezone="AEDT", smtp_server=None, sender=None,
instances_file=None, dry_run=True):
"""Generate mail announcements based on options.
Some files will be generated and written into
~/.cache/hivemind-mailout/<time-stamp> in dry run mode,
which are for operator check and no-dry-run use. They include: 1)
notify.log: run log with all instances info and its email recipients;
2) notification@<project-name>: rendered emails content and
recipients; 3) instances.list: all impacted instances id
:param str template: Template to use for the mailout (Mandatory)
:param str zone: Availability zone affected by outage
:param str ip: Only consider instances with specific ip addresses
:param str nodes: Only target instances from the following Hosts/Nodes
:param str image: Only consider instances with specific image
:param str status: Only consider instances with status
:param str subject: Custom email subject
:param str start_time: Outage start time
:param float duration: Duration of outage in hours
:param str timezone: Timezone
:param str instances_file: Only consider instances listed in file
:param boolean dry_run: By default generate emails without sending out\
use --no-dry-run to send all notifications
:param str smtp_server: Specify the SMTP server
:param str sender: Specify the mail sender
"""
_validate_paramters(start_time, duration, instances_file, template)
config = get_smtp_config(smtp_server, sender)
start_time = datetime.datetime.strptime(start_time, '%H:%M %d-%m-%Y')\
if start_time else None
end_time = start_time + datetime.timedelta(hours=int(duration))\
if (start_time and duration) else None
# find the impacted instances and construct data
if not instances_file:
instances = nova.list_instances(zone=zone, nodes=nodes, ip=ip,
project=project, user=user,
status=status, image=image)
else:
inst = get_instances_from_file(nova.client(), instances_file)
instances = nova.extract_servers_info(inst)
data = populate_data(instances)
# write to logs and generate emails
work_dir = os.path.join(os.path.expanduser('~/.cache/hivemind-mailout'),
datetime.datetime.now().strftime(
"%y-%m-%d_" + "%H:%M:%S"))
print("Creating Outbox: " + work_dir)
os.makedirs(work_dir)
affected = len(data)
if affected:
generate_logs(work_dir, data)
generate_notification_mails(subject, template, data, work_dir,
start_time, end_time, timezone,
zone, affected, nodes)
else:
print("No notification is needed, exit!")
sys.exit(0)
if dry_run:
print("Finish writing email announcement in: " + work_dir)
print("\nOnce you have checked the log file and generated emails")
print("Use the command below to verify emails sending to test user:"******"\n hivemind notification.verify_mailout " + work_dir + " "
+ "SUBJECT" + " " + "[--mailto TEST_ADDRESS]" + " "
+ "[--smtp_server SMTP_SEVRVER]")
print("\nThen rerun the command with --no-dry-run to mail ALL users")
else:
mailout(work_dir, data, subject, config)
make_archive(work_dir)
def freshdesk_mailout(template, zone=None, ip=None, nodes=None, image=None,
status="ALL", project=None, user=None,
subject="Important announcement concerning your "
"instance(s)", start_time=None, duration=None,
timezone="AEDT", instances_file=None,
dry_run=True, record_metadata=False,
metadata_field="notification:fd_ticket",
test_recipient=None):
"""Mailout announcements from freshdesk (Recommended).
Freshdesk ticket per project will be created along with outbound email.
Each mail will notify the first TenantManager and cc all other members.
Once the customer responds to the mail, the reply will be appended to
the ticket and status is changed to Open. Some files will be generated
and written into ~/.cache/hivemind-mailout/freshdesk/<XXXX> in dry run
mode, which are for operator check and no-dry-run use. They include: 1)
notify.log: run log with all instances info and its email recipients;
2) notification@<project-name>: rendered emails content and
recipients; 3) instances.list: all impacted instances id
:param str template: template path to use for the mailout
:param str zone: Availability zone affected by outage
:param str ip: Only consider instances with specific ip addresses
:param str nodes: Only target instances from the following Hosts/Nodes
:param str image: Only consider instances with specific image
:param str status: Only consider instances with status
:param str subject: Custom email subject
:param str start_time: Outage start time
:param float duration: duration of outage in hours
:param str timezone: Timezone
:param str instances_file: Only consider instances listed in file
:param boolean dry_run: by default print info only, use --no-dry-run\
for realsies
:param boolean record_metadata: record the freshdesk ticket URL in\
the nova instance metadata
:param str metadata_field: set the name of the freshdesk ticket URL\
metadata field in the nova instance."
"""
fd_config = security.get_freshdesk_config()
fd = security.get_freshdesk_client(fd_config['domain'],
fd_config['api_key'])
nc = nova.client()
_validate_paramters(start_time, duration, instances_file, template)
start_time = datetime.datetime.strptime(start_time, '%H:%M %d-%m-%Y')\
if start_time else None
end_time = start_time + datetime.timedelta(hours=int(duration))\
if (start_time and duration) else None
work_dir = os.path.expanduser('~/.cache/hivemind-mailout/freshdesk/')
if dry_run:
# find the impacted instances and construct data
if not instances_file:
instances = nova.list_instances(zone=zone, nodes=nodes, ip=ip,
project=project, user=user,
status=status, image=image)
else:
inst = get_instances_from_file(nova.client(), instances_file)
instances = nova.extract_servers_info(inst)
# group by project
data = populate_data(instances)
if not data:
print("No notification needed, exit!")
sys.exit(0)
affected = len(data)
print("\n DRY RUN MODE - only generate notification emails: \n")
# write to logs and generate emails
if not os.path.isdir(work_dir):
os.makedirs(work_dir)
work_dir = tempfile.mkdtemp(dir=work_dir)
print("Creating Outbox: " + work_dir)
print('\nPlease export the environment variable for the no-dry-run: '
'export %s=%s' % ('HIVEMIND_MAILOUT_FRESHDESK', work_dir))
# render email content
generate_logs(work_dir, data)
generate_notification_mails(subject, template, data, work_dir,
start_time, end_time, timezone,
zone, affected, nodes)
else:
work_dir = os.environ.get('HIVEMIND_MAILOUT_FRESHDESK')
if not work_dir or not os.path.isdir(work_dir):
print('Workdir environment variable is not found!')
print('Please run the command without --no-dry-run and '
'export environment variable as prompted!')
sys.exit(0)
email_files = [name for name in os.listdir(work_dir)
if os.path.isfile(os.path.join(work_dir, name))
and "notification@" in name]
if test_recipient:
print('You have specified the test_recipient option, '
'all the emails will be sent to %s' % test_recipient)
query = '\nYou are running notification script in no-dry-run mode'\
'\nIt will use previously generated emails under %s\n'\
'Make sure the contents are all good before you do next step'\
'\nOne outbounding email will create a separate ticket. '\
'Be cautious since it could generate massive tickets!!!\n'\
'There are %s tickets to be created, it takes roughly %s min '\
'to finish all the creation, still continue?'
if not query_yes_no(query % (work_dir, len(email_files),
len(email_files) / 60 + 1), default='no'):
sys.exit(1)
subjectfd = "[Nectar Notice] " + subject
for email_file in email_files:
print('\nCreating new Freshdesk ticket')
with open(os.path.join(work_dir, email_file), 'rb') as f:
email = yaml.load(f)
addresses = email['Sendto'].split(',')
toaddress = addresses.pop(0)
if test_recipient:
toaddress = test_recipient
addresses = [test_recipient]
ticket = fd.tickets.create_outbound_email(
description=email['Body'],
subject=subjectfd,
email=toaddress,
cc_emails=addresses,
email_config_id=int(fd_config['email_config_id']),
group_id=int(fd_config['group_id']),
priority=2,
status=5, # set ticket initial status is closed
tags=['notification'])
ticket_id = ticket.id
# Use friendly domain name if using prod
if fd.domain == 'dhdnectar.freshdesk.com':
domain = 'support.ehelp.edu.au'
else:
domain = fd.domain
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(domain, ticket_id)
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
if record_metadata:
# Record the ticket URL in the server metadata
for server in email[4]:
nc.servers.set_meta(server['id'],
{metadata_field: ticket_url})
# delay for freshdesk api rate limit consideration
time.sleep(1)
# make achive the outbox folder
print('Make archive after the mailout for %s' % work_dir)
make_archive(work_dir)
def generate_instance_info(instance_id, style=None):
nc = nova.client()
kc = keystone.client()
gc = glance.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error("Instance {} not found".format(instance_id))
info = instance._info.copy()
for network_label, address_list in instance.networks.items():
info['%s network' % network_label] = ', '.join(address_list)
flavor = info.get('flavor', {})
flavor_id = flavor.get('id', '')
try:
info['flavor'] = '%s (%s)' % (nova.get_flavor(nc, flavor_id).name,
flavor_id)
except Exception:
info['flavor'] = '%s (%s)' % ("Flavor not found", flavor_id)
# Image
image = info.get('image', {})
if image:
image_id = image.get('id', '')
try:
img = gc.images.get(image_id)
nectar_build = img.get('nectar_build', 'N/A')
info['image'] = ('%s (%s, NeCTAR Build %s)'
% (img.name, img.id, nectar_build))
except Exception:
info['image'] = 'Image not found (%s)' % image_id
else: # Booted from volume
info['image'] = "Attempt to boot from volume - no image supplied"
# Tenant
project_id = info.get('tenant_id')
if project_id:
try:
project = keystone.get_project(kc, project_id)
info['project_id'] = '%s (%s)' % (project.name, project.id)
except Exception:
pass
# User
user_id = info.get('user_id')
if user_id:
try:
user = keystone.get_user(kc, user_id)
info['user_id'] = '%s (%s)' % (user.name, user.id)
except Exception:
pass
# Remove stuff
info.pop('links', None)
info.pop('addresses', None)
info.pop('hostId', None)
info.pop('security_groups', None)
return _format_instance(info, style=style)
def quota_reversions(infile=None, id_col=1, cores_col=2,
instances_col=3, outfile=None):
tenants = {};
if infile != None:
with open(infile, 'rb') as csvfile:
for row in csv.reader(csvfile, delimiter=','):
try:
tenants[row[id_col]] = [int(row[cores_col]),
int(row[instances_col])]
except:
continue
allocations = _get_current_allocations()
nova_api = hm_nova.client()
flavors = _get_flavor_map(nova_api)
updates = []
for uuid in allocations.keys():
alloc = allocations[uuid]
if len(tenants) > 0:
# Figure out what the quota should have been prior to
# the adjustment
if uuid in tenants:
deltas = tenants[uuid]
expected = {
'core_quota': alloc['core_quota'] + deltas[0],
'instance_quota': alloc['instance_quota'] + deltas[1],
'ram_quota': (alloc['ram_quota'] + deltas[0] * 4) * 1024
}
else:
# No data for this tenant
continue
else:
expected = None
try:
quotas = nova_api.quotas.get(uuid)
except:
continue
alloc['expected'] = expected
alloc['nova_quotas'] = quotas
alloc['deltas'] = deltas
updates.append(alloc)
# If the alloc and current quotas match, don't touch them
if (quotas.instances == alloc['instance_quota'] \
and quotas.ram == alloc['ram_quota'] * 1024 \
and quotas.cores == alloc['core_quota']):
alloc['update'] = 'no - quotas match'
continue
usage = _get_usage(nova_api, flavors, uuid)
alloc['nova_usage'] = usage
# If the usage is greater than the allocated quotas, don't touch them
if (usage['instances'] > alloc['instance_quota']
or usage['vcpus'] > alloc['core_quota']
or usage['ram'] > alloc['ram_quota'] * 1024):
alloc['update'] = 'no - over-quota usage'
continue
# If the difference between the nova quotas don't match the
# expected quotas (i.e. alloc + deltas), don't touch them
if (expected != None
and (expected['instance_quota'] != quotas.instances
or expected['ram_quota'] != quotas.ram
or expected['core_quota'] != quotas.cores)):
alloc['update'] = 'no - deltas wrong'
continue
alloc['update'] = 'yes'
fields_to_report = [
("Tenant ID", lambda x: x['tenant_uuid']),
("Tenant Name", lambda x: x['tenant_name']),
("Instances", lambda x: x['instance_quota']),
("vCPU quota", lambda x: x['core_quota']),
("Memory", lambda x: x['ram_quota'] * 1024),
("Nova Instance quota", lambda x: x['nova_quotas'].instances),
("Nova vCPU quota", lambda x: x['nova_quotas'].cores),
("Nova Memory quota", lambda x: x['nova_quotas'].ram),
("Nova Instance usage",
lambda x: x['nova_usage']['instances'] if 'nova_usage' in x else ''),
("Nova vCPU usage",
lambda x: x['nova_usage']['vcpus'] if 'nova_usage' in x else ''),
("Nova Memory usage",
lambda x: x['nova_usage']['ram'] if 'nova_usage' in x else ''),
("Instance delta", lambda x: x['deltas'][1] if 'deltas' in x else ''),
("vCPU delta", lambda x: x['deltas'][0] if 'deltas' in x else ''),
("Update", lambda x: x['update'])
]
csv_output(map(lambda x: x[0], fields_to_report),
map(lambda alloc: map(
lambda y: y[1](alloc),
fields_to_report),
updates),
filename=outfile)
def lock_instance(instance_id, dry_run=True):
"""pause and lock an instance"""
if dry_run:
print('Running in dry-run mode (use --no-dry-run for realsies)')
fd_config = get_freshdesk_config()
fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key'])
nc = nova.client()
kc = keystone.client()
try:
instance = nc.servers.get(instance_id)
except n_exc.NotFound:
error('Instance {} not found'.format(instance_id))
# Pause and lock instance
if dry_run:
if instance.status != 'ACTIVE':
print('Instance state {}, will not pause'.format(instance.status))
else:
print('Would pause and lock instance {}'.format(instance_id))
else:
if instance.status != 'ACTIVE':
print('Instance not in ACTIVE state ({}), skipping'
.format(instance.status))
else:
print('Pausing instance {}'.format(instance_id))
instance.pause()
print('Locking instance {}'.format(instance_id))
instance.lock()
# Process ticket
ticket_id = None
ticket_url = instance.metadata.get('security_ticket')
if ticket_url:
print('Found existing ticket: {}'.format(ticket_url))
ticket_id = int(ticket_url.split('/')[-1])
if dry_run:
print('Would set ticket #{} status to open/urgent'
.format(ticket_id))
else:
# Set ticket status=waiting for customer, priority=urgent
print('Setting ticket #{} status to open/urgent'.format(ticket_id))
fd.tickets.update_ticket(ticket_id, status=6, priority=4)
else:
project = keystone.get_project(kc, instance.tenant_id)
user = keystone.get_user(kc, instance.user_id)
email = user.email or '*****@*****.**'
name = getattr(user, 'full_name', email)
cc_emails = get_tenant_managers_emails(kc, instance)
# Create ticket if none exist, and add instance info
subject = 'Security incident for instance {} ({})'.format(
instance.name, instance_id)
body = '<br />\n'.join([
'Dear Nectar Research Cloud User, ',
'',
'',
'We have reason to believe that cloud instance: '
'<b>{} ({})</b>'.format(instance.name, instance_id),
'in the project <b>{}</b>'.format(project.name),
'created by <b>{}</b>'.format(email),
'has been involved in a security incident, and has been locked.',
'',
'We have opened this helpdesk ticket to track the details and ',
'the progress of the resolution of this issue.',
'',
'Please reply to this email if you have any questions or ',
'concerns.',
'',
'Thanks, ',
'Nectar Research Cloud Team'
])
if dry_run:
print('Would create ticket with details:')
print(' To: {} <{}>'.format(name, email))
print(' CC: {}'.format(', '.join(cc_emails)))
print(' Subject: {}'.format(subject))
print('Would add instance details to ticket:')
print(generate_instance_info(instance_id))
print(generate_instance_sg_rules_info(instance_id))
else:
print('Creating new Freshdesk ticket')
ticket = fd.tickets.create_outbound_email(
name=name,
description=body,
subject=subject,
email=email,
cc_emails=cc_emails,
email_config_id=int(fd_config['email_config_id']),
group_id=int(fd_config['group_id']),
priority=4,
status=2,
tags=['security'])
ticket_id = ticket.id
# Use friendly domain name if using prod
if fd.domain == 'dhdnectar.freshdesk.com':
domain = 'support.ehelp.edu.au'
else:
domain = fd.domain
ticket_url = 'https://{}/helpdesk/tickets/{}'\
.format(domain, ticket_id)
nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
print('Ticket #{} has been created: {}'
.format(ticket_id, ticket_url))
# Add a private note with instance details
print('Adding instance information to ticket')
instance_info = generate_instance_info(instance_id, style='html')
sg_info = generate_instance_sg_rules_info(instance_id,
style='html')
body = '<br/><br/>'.join([instance_info, sg_info])
fd.comments.create_note(ticket_id, body)
