def get_project_usage_csv(start_date=None, end_date=None, filename=None, sslwarnings=False): """Get accumulated instance usage statistics for all projects. Date strings should be ISO 8601 to minute precision without timezone information. """ ssl_warnings(enabled=sslwarnings) assert start_date and end_date start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M") end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M") keystone = hm_keystone.client_session(version=3) nova = hm_nova.client() tenants = {x.id: x for x in keystone.projects.list()} headings = ["Tenant ID", "Tenant Name", "Instance count", "Instance hours", "vCPU hours", "Memory Hours (MB)", "Disk hours (GB)"] usage = map(lambda u: [ u.tenant_id, tenants[u.tenant_id].name if u.tenant_id in tenants else None, len(u.server_usages), u.total_hours, u.total_vcpus_usage, u.total_memory_mb_usage, u.total_local_gb_usage], nova.usage.list(start, end, detailed=True)) csv_output(headings, usage, filename=filename)
def generate_instance_info(instance_id, style=None): nc = nova.client() kc = keystone.client() gc = glance.get_glance_client(kc) try: instance = nc.servers.get(instance_id) except n_exc.NotFound: error("Instance {} not found".format(instance_id)) info = instance._info.copy() for network_label, address_list in instance.networks.items(): info["%s network" % network_label] = ", ".join(address_list) flavor = info.get("flavor", {}) flavor_id = flavor.get("id", "") try: info["flavor"] = "%s (%s)" % (nova.get_flavor(nc, flavor_id).name, flavor_id) except Exception: info["flavor"] = "%s (%s)" % ("Flavor not found", flavor_id) # Image image = info.get("image", {}) if image: image_id = image.get("id", "") try: img = gc.images.get(image_id) nectar_build = img.properties.get("nectar_build", "N/A") info["image"] = "%s (%s, NeCTAR Build %s)" % (img.name, img.id, nectar_build) except Exception: info["image"] = "Image not found (%s)" % image_id else: # Booted from volume info["image"] = "Attempt to boot from volume - no image supplied" # Tenant tenant_id = info.get("tenant_id") if tenant_id: try: tenant = keystone.get_tenant(kc, tenant_id) info["tenant_id"] = "%s (%s)" % (tenant.name, tenant.id) except Exception: pass # User user_id = info.get("user_id") if user_id: try: user = keystone.get_user(kc, user_id) info["user_id"] = "%s (%s)" % (user.name, user.id) except Exception: pass # Remove stuff info.pop("links", None) info.pop("addresses", None) info.pop("hostId", None) info.pop("security_groups", None) return _format_instance(info, style=style)
def public_audit(): """Print usage information about all public images """ gc = get_glance_client(keystone.client(), api_version=2) nc = nova.client() db = nova.db_connect() # The visibility filter doesn't seem to work... so we filter them out again images = gc.images.list(visibility='public') public = [i for i in images if i['visibility'] == 'public'] table = PrettyTable(["ID", "Name", "Num running instances", "Boot count", "Last Boot"]) for i in public: sql = select([nova.instances_table]) where = [nova.instances_table.c.image_ref.like(i['id'])] sql = sql.where(*where).order_by(desc('created_at')) image_instances = db.execute(sql).fetchall() boot_count = len(image_instances) if boot_count > 0: last_boot = image_instances[0].created_at else: last_boot = 'Never' instances = nova.all_servers(nc, image=i['id']) table.add_row([i['id'], i['name'], len(instances), boot_count, last_boot]) print(table.get_string(sortby="Num running instances", reversesort=True))
def get_instance_usage_csv(start_date=None, end_date=None, filename=None): """Get instance usage statistics for all projects. Date strings should be ISO 8601 to minute precision without timezone information. """ assert start_date and end_date start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M") end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M") keystone = hm_keystone.client() nova = hm_nova.client() tenants = {x.id: x for x in keystone.tenants.list()} headings = ["Tenant ID", "Tenant Name", "Instance count", "Instance hours", "vCPU hours", "Memory Hours (MB)", "Disk hours (GB)"] usage = map(lambda u: [ u.tenant_id, tenants[u.tenant_id].name if u.tenant_id in tenants else None, len(u.server_usages), u.total_hours, u.total_vcpus_usage, u.total_memory_mb_usage, u.total_local_gb_usage], nova.usage.list(start, end, detailed=True)) csv_output(headings, usage, filename=filename)
def link_account(existing_email, new_email): db = connect() ids = keystone_ids_from_email(db, existing_email) new_email_ids = keystone_ids_from_email(db, new_email) if len(ids) != 1: print('User has multiple accounts with email %s' % existing_email) return user_id = ids[0] orphan_user_id = new_email_ids[0] print('%s: %s' % (existing_email, user_id)) print('%s: %s' % (new_email, orphan_user_id)) if user_id == orphan_user_id: print('Those accounts are already linked') return client = keystone.client() user = client.users.get(orphan_user_id) project = client.projects.get(user.default_project_id) servers = nova.client().servers.list(search_opts={ 'all_tenants': True, 'project_id': project.id}) if len(servers): print('Soon to be orphaned project has active instances.') print('Advise user to terminate them.') return print() print('Confirm that you want to:') print(' - Link %s to account %s' % (new_email, existing_email)) print(' - Disable orphan Keystone project %s' % (project.name)) print(' - Disable orphan Keystone user %s' % (user.name)) print() response = raw_input('(yes/no): ') if response != 'yes': return print('Linking account.') sql = (update(users) .where(users.c.email == new_email) .values(user_id=user_id)) result = db.execute(sql) if result.rowcount == 0: print('Something went wrong.') return print('Disabling orphaned Keystone project %s (%s).' % ( project.name, project.id)) client.projects.update(project.id, enabled=False) print('Disabling orphaned Keystone user %s (%s).' % (user.name, user.id)) client.users.update(user.id, enabled=False, name="%s-disabled" % user.name) print('All done.')
def get_instance_usage_csv(start_date=None, end_date=None, filename=None, sslwarnings=False): """Get individual instance usage for all projects, including tenant and availability zones. Date strings should be ISO 8601 to minute precision without timezone information. """ ssl_warnings(enabled=sslwarnings) assert start_date and end_date start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M") end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M") keystone = hm_keystone.client_session(version=3) nova = hm_nova.client() tenants = {x.id: x for x in keystone.projects.list()} usage = [] for u in nova.usage.list(start, end, detailed=True): tenant_id = u.tenant_id tenant_name = tenants[tenant_id].name if tenant_id in tenants else None # The Nova API doesn't allow "show" on deleted instances, but # we can get the info using "list --deleted". The problem is # figuring out how to avoid retrieving irrelevant instances, # and at the same time how to avoid too many requests. # # Attempt #1 - use the tenant_id and the instance's name to # focus queries. # Attempt #2 - as #1, but after N lookups by name for a tenant, # just fetch all of the deleted instances. cache = {} try: for iu in u.server_usages: name = iu['name'] instance_id = iu['instance_id'] instance = None if iu['state'] == 'terminated' or iu['state'] == 'deleted': instance = _get_deleted_instance(cache, nova, u.tenant_id, name, instance_id) else: try: instance = nova.servers.get(instance_id).to_dict() except: print 'Cannot find instance {0} in {1}' \ .format(instance_id, u.tenant_id) if instance is None: instance = {'OS-EXT-AZ:availability_zone': 'unknown'} usage.append([tenant_id, tenant_name, instance_id, name, iu['state'], iu['flavor'], iu['hours'], iu['vcpus'], iu['memory_mb'], iu['local_gb'], instance['OS-EXT-AZ:availability_zone']]) except: traceback.print_exc(file=sys.stdout) headings = ["Tenant ID", "Tenant Name", "Instance id", "Instance name", "Instance state", "Flavour", "Instance hours", "vCPUs", "Memory (MB)", "Disk (GB)", "AZ"] csv_output(headings, usage, filename=filename)
def link_account(existing_email, new_email): db = connect() ids = keystone_ids_from_email(db, existing_email) new_email_ids = keystone_ids_from_email(db, new_email) if len(ids) != 1: print 'User has multiple accounts with email %s' % existing_email return user_id = ids[0] orphan_user_id = new_email_ids[0] print '%s: %s' % (existing_email, user_id) print '%s: %s' % (new_email, orphan_user_id) if user_id == orphan_user_id: print 'Those accounts are already linked' return client = keystone.client() user = client.users.get(orphan_user_id) project = client.tenants.get(user.tenantId) servers = nova.client().servers.list(search_opts={ 'all_tenants': True, 'project_id': project.id}) if len(servers): print 'Soon to be orphaned project has active instances.' print 'Advise user to terminate them.' return print print 'Confirm that you want to:' print ' - Link %s to account %s' % (new_email, existing_email) print ' - Delete orphan Keystone project %s' % (project.name) print ' - Delete orphan Keystone user %s' % (user.name) print response = raw_input('(yes/no): ') if response != 'yes': return print 'Linking account.' sql = (update(users) .where(users.c.email == new_email) .values(user_id=user_id)) result = db.execute(sql) if result.rowcount == 0: print 'Something went wrong.' return print 'Deleting orphaned Keystone project %s (%s).' % ( project.name, project.id) client.tenants.delete(project.id) print 'Deleting orphaned Keystone user %s (%s).' % (user.name, user.id) client.users.delete(user.id) print 'All done.'
def sync_vm_rules(project_id=None): """Sync security groups for the given instance UUID (-I)""" if not env.instance_uuid and project_id is None: error("No instance ID or project specified.") if env.instance_uuid: uuid = env.instance_uuid nova_client = client() server = nova_client.servers.get(uuid) project_id = server.tenant_id with show('stdout', 'stderr'): run('nova-manage project sync_secgroups %s' % project_id)
def sync_vm_rules(project_id=None): """Sync security groups for the given instance UUID (-I)""" if not env.instance_uuid and project_id is None: error("No instance ID or project specified.") if env.instance_uuid: uuid = env.instance_uuid nova_client = client() server = nova_client.servers.get(uuid) project_id = server.tenant_id with show('stdout', 'stderr'): run('nova-manage project sync_secgroups %s' % project_id)
def link_account(existing_email, new_email): db = connect() ids = keystone_ids_from_email(db, existing_email) new_email_ids = keystone_ids_from_email(db, new_email) if len(ids) != 1: print("User has multiple accounts with email %s" % existing_email) return user_id = ids[0] orphan_user_id = new_email_ids[0] print("%s: %s" % (existing_email, user_id)) print("%s: %s" % (new_email, orphan_user_id)) if user_id == orphan_user_id: print("Those accounts are already linked") return client = keystone.client() user = client.users.get(orphan_user_id) project = client.tenants.get(user.tenantId) servers = nova.client().servers.list(search_opts={"all_tenants": True, "project_id": project.id}) if len(servers): print("Soon to be orphaned project has active instances.") print("Advise user to terminate them.") return print() print("Confirm that you want to:") print(" - Link %s to account %s" % (new_email, existing_email)) print(" - Delete orphan Keystone project %s" % (project.name)) print(" - Delete orphan Keystone user %s" % (user.name)) print() response = raw_input("(yes/no): ") if response != "yes": return print("Linking account.") sql = update(users).where(users.c.email == new_email).values(user_id=user_id) result = db.execute(sql) if result.rowcount == 0: print("Something went wrong.") return print("Deleting orphaned Keystone project %s (%s)." % (project.name, project.id)) client.tenants.delete(project.id) print("Deleting orphaned Keystone user %s (%s)." % (user.name, user.id)) client.users.delete(user.id) print("All done.")
def compare_quotas(name_or_id=None): """Compare the allocation and quota information for a tenant """ if name_or_id == None: print 'A tenant name or id is required' return keystone_api = hm_keystone.client_session(version=3) try: tenant = hm_keystone.get_tenant(keystone_api, name_or_id) except: print 'Tenant {0} not found in keystone'.format(name_or_id) return nova_api = hm_nova.client() quotas = nova_api.quotas.get(tenant.id) print 'nova quotas: instances {0}, cores {1}, ram {2}'.format( quotas.instances, quotas.cores, quotas.ram / 1024) usage = _get_usage(nova_api, _get_flavor_map(nova_api), tenant.id) print 'nova usage: instances {0}, cores {1}, ram {2}'.format( usage['instances'], usage['vcpus'], usage['ram'] / 1024) allocations_api = NectarApiSession() allocations = allocations_api.get_allocations(); tenant_allocations = filter(lambda x: x['tenant_uuid'] == tenant.id and \ (x['status'] == 'A' or x['status'] == 'X'), allocations) if len(tenant_allocations) == 0: print 'No approved allocation records for tenant {0} / {1}'.format( tenant.id, tenant.name) return tenant_allocations.sort(key=lambda alloc: alloc['modified_time']) current_allocation = tenant_allocations[-1] format = '{0} mismatch: allocated {1}, nova {2}, used {3}' if current_allocation['instance_quota'] != quotas.instances: print format.format('Instance quota', current_allocation['instance_quota'], quotas.instances, usage['instances']) if current_allocation['core_quota'] != quotas.cores: print format.format('VCPU quota', current_allocation['core_quota'], quotas.cores, usage['vcpus']) if current_allocation['ram_quota'] * 1024 != quotas.ram: print format.format('RAM quota', current_allocation['ram_quota'] * 1024, quotas.ram, usage['ram'])
def unlock_instance(instance_id, dry_run=True): """unlock an instance""" if dry_run: print('Running in dry-run mode (use --no-dry-run for realsies)') fd = get_freshdesk_client() nc = nova.client() try: instance = nc.servers.get(instance_id) except n_exc.NotFound: error('Instance {} not found'.format(instance_id)) ticket_id = None ticket_url = instance.metadata.get('security_ticket') if ticket_url: print('Found ticket: {}'.format(ticket_url)) ticket_id = int(ticket_url.split('/')[-1]) else: if not dry_run: error('No ticket found in instance metadata!') if dry_run: print('Would unpause and unlock instance {}'.format(instance_id)) print('Would reply to ticket') print('Would resolve ticket') else: if instance.status != 'PAUSED': print('Instance not paused') else: print('Unpausing instance {}'.format(instance_id)) instance.unpause() print('Unlocking instance {}'.format(instance_id)) instance.unlock() # Add reply to user email_addresses = get_ticket_recipients(instance) print('Replying to ticket with action details') action = 'Instance <b>{} ({})</b> has been <b>unpaused and '\ 'unlocked</b>'.format(instance.name, instance_id) fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses) # Set ticket status=resolved print('Setting ticket #{} status to resolved'.format(ticket_id)) fd.tickets.update_ticket(ticket_id, status=4)
def public_audit(): """Print usage information about all public images """ gc = client() nc = nova.client() db = nova.db_connect() # The visibility filter doesn't seem to work... so we filter them out again images = gc.images.list(visibility='public') public = [i for i in images if i['visibility'] == 'public'] table = PrettyTable(["ID", "Name", "Official", "Build", "Running", "Boots", "Last Boot"]) table.align = 'l' table.align['Running'] = 'r' table.align['Boots'] = 'r' for i in public: sql = select([nova.instances_table]) where = [nova.instances_table.c.image_ref.like(i.id)] sql = sql.where(*where).order_by(desc('created_at')) image_instances = db.execute(sql).fetchall() boot_count = len(image_instances) last_boot = 'Never' if boot_count > 0: last_boot = image_instances[0].created_at instances = nova.all_servers(nc, image=i['id']) # NeCTAR-Images, NeCTAR-Images-Archive official_projects = ['28eadf5ad64b42a4929b2fb7df99275c', 'c9217cb583f24c7f96567a4d6530e405'] if i.owner in official_projects: official = 'Y' else: official = 'N' name = i.get('name', 'n/a') build = i.get('nectar_build', 'n/a') num = len(instances) if instances else 0 table.add_row([i.id, name, official, build, num, boot_count, last_boot]) print(table.get_string(sortby="Running", reversesort=True))
def official_audit(): """Print usage information about official images """ data = {} gc = client() nc = nova.client() db = nova.db_connect() images = [] # NeCTAR-Images, NeCTAR-Images-Archive official_projects = ['28eadf5ad64b42a4929b2fb7df99275c', 'c9217cb583f24c7f96567a4d6530e405'] for project in official_projects: images += list(gc.images.list(filters={'owner': project})) table = PrettyTable(["Name", "Running", "Boots"]) table.align = 'l' table.align['Running'] = 'r' table.align['Boots'] = 'r' for i in images: sql = select([nova.instances_table]) where = [nova.instances_table.c.image_ref.like(i.id)] sql = sql.where(*where).order_by(desc('created_at')) image_instances = db.execute(sql).fetchall() boot_count = len(image_instances) instances = nova.all_servers(nc, image=i['id']) if i.owner in official_projects or not i.owner: if i.name in data: data[i.name]['running'] += len(instances) data[i.name]['boots'] += boot_count else: data[i.name] = {'running': len(instances), 'boots': boot_count} for d in data.iteritems(): table.add_row([d[0], d[1]['running'], d[1]['boots']]) print(table.get_string(sortby="Running", reversesort=True))
def vm_rules(): if not env.instance_uuid: error("No instance_uuid specified.") uuid = env.instance_uuid nova_client = client() server = nova_client.servers.get(uuid) host = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname') libvirt_server = [s for s in chain(*execute(list_instances, host=host).values()) if s['uuid'] == uuid][0] for vm, rules in chain.from_iterable(map(dict.items, execute(parse_rules, host=host).values())): if vm != str(libvirt_server['nova_id']): continue table = PrettyTable(['Target', 'Protocol', 'Source', 'Destination', 'Filter']) for rule in rules: table.add_row([rule['target'], rule['protocol'], rule['source'], rule['destination'], rule['filter']]) puts("\n%s\n%s\n" % (server.id, str(table)))
def crosscheck_quotas(filename=None): """Cross-check allocation and quota information for all tenants """ allocations = _get_current_allocations() nova_api = hm_nova.client() missing = [] mismatches = [] for uuid in allocations.keys(): alloc = allocations[uuid] try: quotas = nova_api.quotas.get(uuid) except: missing.append(alloc) continue if (quotas.instances != alloc['instance_quota'] \ or quotas.ram != alloc['ram_quota'] * 1024 \ or quotas.cores != alloc['core_quota']): alloc['nova_quotas'] = quotas mismatches.append(alloc) print '{0} allocations, {1} missing tenants, {2} quota mismatches'.format( len(allocations), len(missing), len(mismatches)) fields_to_report = [ ("Tenant ID", lambda x: x['tenant_uuid']), ("Tenant Name", lambda x: x['tenant_name']), ("Modified time", lambda x: x['modified_time']), ("Instances", lambda x: x['instance_quota']), ("Nova instances", lambda x: x['nova_quotas'].instances), ("vCPU quota", lambda x: x['core_quota']), ("Nova vCPU quota", lambda x: x['nova_quotas'].cores), ("RAM quota", lambda x: x['ram_quota'] * 1024), ("Nova RAM quota", lambda x: x['nova_quotas'].ram) ] csv_output(map(lambda x: x[0], fields_to_report), map(lambda alloc: map( lambda y: y[1](alloc), fields_to_report), mismatches), filename=filename)
def delete_instance(instance_id, dry_run=True): """delete an instance""" if dry_run: print('Running in dry-run mode (use --no-dry-run for realsies)') fd_config = get_freshdesk_config() fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key']) nc = nova.client() try: instance = nc.servers.get(instance_id) except n_exc.NotFound: error('Instance {} not found'.format(instance_id)) ticket_id = None ticket_url = instance.metadata.get('security_ticket') if ticket_url: print('Found ticket: {}'.format(ticket_url)) ticket_id = int(ticket_url.split('/')[-1]) else: if not dry_run: error('No ticket found in instance metadata!') # DELETE!!! if dry_run: print('Would delete instance {}'.format(instance_id)) print('Would reply to ticket') print('Would resolve ticket') else: print('Deleting instance {})'.format(instance_id)) instance.delete() # Add reply to user print('Updating ticket with action') action = 'Instance <b>{} ({})</b> has been <b>deleted.</b>'\ .format(instance.name, instance_id) fd.comments.create_reply(ticket_id, action) # Set ticket status=resolved print('Resolving ticket #{}'.format(ticket_id)) fd.tickets.update_ticket(ticket_id, status=4)
def crosscheck_usage(filename=None): """Cross-check that allocation and instantaneous usage information for all tenants """ allocations = _get_current_allocations() nova_api = hm_nova.client() flavors = _get_flavor_map(nova_api) missing = [] mismatches = [] for uuid in allocations.keys(): alloc = allocations[uuid] usage = _get_usage(nova_api, flavors, uuid) if (usage['instances'] > alloc['instance_quota'] or usage['vcpus'] > alloc['core_quota'] or usage['ram'] > alloc['ram_quota'] * 1024): alloc['nova_usage'] = usage mismatches.append(alloc) print '{0} allocations, {1} missing tenants, {2} usage mismatches'.format( len(allocations), len(missing), len(mismatches)) fields_to_report = [ ("Tenant ID", lambda x: x['tenant_uuid']), ("Tenant Name", lambda x: x['tenant_name']), ("Modified time", lambda x: x['modified_time']), ("Instances", lambda x: x['instance_quota']), ("Nova instances", lambda x: x['nova_usage']['instances']), ("vCPU quota", lambda x: x['core_quota']), ("Nova vCPU usage", lambda x: x['nova_usage']['vcpus']), ("RAM quota", lambda x: x['ram_quota'] * 1024), ("Nova RAM usage", lambda x: x['nova_usage']['ram']) ] csv_output(map(lambda x: x[0], fields_to_report), map(lambda alloc: map( lambda y: y[1](alloc), fields_to_report), mismatches), filename=filename)
def vm_rules(): if not env.instance_uuid: error("No instance_uuid specified.") uuid = env.instance_uuid nova_client = client() server = nova_client.servers.get(uuid) host = getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname') libvirt_server = [ s for s in chain(*execute(list_instances, host=host).values()) if s['uuid'] == uuid ][0] for vm, rules in chain.from_iterable( map(dict.items, execute(parse_rules, host=host).values())): if vm != str(libvirt_server['nova_id']): continue table = PrettyTable( ['Target', 'Protocol', 'Source', 'Destination', 'Filter']) for rule in rules: table.add_row([ rule['target'], rule['protocol'], rule['source'], rule['destination'], rule['filter'] ]) puts("\n%s\n%s\n" % (server.id, str(table)))
def lock_instance(instance_id, dry_run=True): """pause and lock an instance""" if dry_run: print('Running in dry-run mode (use --no-dry-run for realsies)') fd = get_freshdesk_client() nc = nova.client() kc = keystone.client() try: instance = nc.servers.get(instance_id) except n_exc.NotFound: error('Instance {} not found'.format(instance_id)) ticket_id = None ticket_url = instance.metadata.get('security_ticket') if ticket_url: print('Found existing ticket: {}'.format(ticket_url)) ticket_id = int(ticket_url.split('/')[-1]) if dry_run: print('Would set ticket #{} status to open/urgent' .format(ticket_id)) else: # Set ticket status=waiting for customer, priority=urgent print('Setting ticket #{} status to open/urgent'.format(ticket_id)) fd.tickets.update_ticket(ticket_id, status=6, priority=4) else: tenant = keystone.get_tenant(kc, instance.tenant_id) user = keystone.get_user(kc, instance.user_id) email_addresses = get_ticket_recipients(instance) # Create ticket if none exist, and add instance info subject = 'Security incident for instance {}'.format(instance_id) body = '<br />\n'.join([ 'Dear NeCTAR Research Cloud User, ', '', '', 'We have reason to believe that cloud instance: ' '<b>{} ({})</b>'.format(instance.name, instance.id), 'in the project <b>{}</b>'.format(tenant.name), 'created by <b>{}</b>'.format(user.email), 'has been involved in a security incident.', '', 'We have opened this helpdesk ticket to track the details and ', 'the progress of the resolution of this issue.', '', 'Please reply to this email if you have any questions or ', 'concerns.', '', 'Thanks, ', 'NeCTAR Research Cloud Team' ]) if dry_run: print('Would create ticket with details:') print(' To: {}'.format(email_addresses)) print(' Subject: {}'.format(subject)) print('Would add instance details to ticket:') print(generate_instance_info(instance_id)) print(generate_instance_sg_rules_info(instance_id)) else: print('Creating new Freshdesk ticket') ticket = fd.tickets.create_ticket( description=body, subject=subject, email='*****@*****.**', cc_emails=email_addresses, priority=4, status=6, tags=['security']) ticket_id = ticket.id ticket_url = 'https://{}/helpdesk/tickets/{}'\ .format(fd.domain, ticket_id) nc.servers.set_meta(instance_id, {'security_ticket': ticket_url}) print('Ticket #{} has been created: {}' .format(ticket_id, ticket_url)) # Add a private note with instance details print('Adding instance information to ticket') instance_info = generate_instance_info(instance_id, style='html') sg_info = generate_instance_sg_rules_info(instance_id, style='html') body = '<br/><br/>'.join([instance_info, sg_info]) fd.comments.create_note(ticket_id, body) if dry_run: if instance.status != 'ACTIVE': print('Instance state {}, will not pause'.format(instance.status)) else: print('Would pause and lock instance {}'.format(instance_id)) print('Would update ticket with action') else: # Pause and lock if instance.status != 'ACTIVE': print('Instance not in ACTIVE state ({}), skipping' .format(instance.status)) else: print('Pausing instance {}'.format(instance_id)) instance.pause() print('Locking instance {}'.format(instance_id)) instance.lock() # Add reply to user email_addresses = get_ticket_recipients(instance) print('Replying to ticket with action details') action = 'Instance <b>{} ({})</b> has been <b>paused and locked</b> '\ 'pending further investigation'\ .format(instance.name, instance_id) fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
def announcement_mailout(template, zone=None, ip=None, nodes=None, image=None, status="ALL", project=None, user=None, subject="Important announcement concerning your " "instance(s)", start_time=None, duration=0, timezone="AEDT", smtp_server=None, sender=None, instances_file=None, dry_run=True): """Generate mail announcements based on options. Some files will be generated and written into ~/.cache/hivemind-mailout/<time-stamp> in dry run mode, which are for operator check and no-dry-run use. They include: 1) notify.log: run log with all instances info and its email recipients; 2) notification@<project-name>: rendered emails content and recipients; 3) instances.list: all impacted instances id :param str template: Template to use for the mailout (Mandatory) :param str zone: Availability zone affected by outage :param str ip: Only consider instances with specific ip addresses :param str nodes: Only target instances from the following Hosts/Nodes :param str image: Only consider instances with specific image :param str status: Only consider instances with status :param str subject: Custom email subject :param str start_time: Outage start time :param float duration: Duration of outage in hours :param str timezone: Timezone :param str instances_file: Only consider instances listed in file :param boolean dry_run: By default generate emails without sending out\ use --no-dry-run to send all notifications :param str smtp_server: Specify the SMTP server :param str sender: Specify the mail sender """ _validate_paramters(start_time, duration, instances_file, template) config = get_smtp_config(smtp_server, sender) start_time = datetime.datetime.strptime(start_time, '%H:%M %d-%m-%Y')\ if start_time else None end_time = start_time + datetime.timedelta(hours=int(duration))\ if (start_time and duration) else None # find the impacted instances and construct data if not instances_file: instances = nova.list_instances(zone=zone, nodes=nodes, ip=ip, project=project, user=user, status=status, image=image) else: inst = get_instances_from_file(nova.client(), instances_file) instances = nova.extract_servers_info(inst) data = populate_data(instances) # write to logs and generate emails work_dir = os.path.join(os.path.expanduser('~/.cache/hivemind-mailout'), datetime.datetime.now().strftime( "%y-%m-%d_" + "%H:%M:%S")) print("Creating Outbox: " + work_dir) os.makedirs(work_dir) affected = len(data) if affected: generate_logs(work_dir, data) generate_notification_mails(subject, template, data, work_dir, start_time, end_time, timezone, zone, affected, nodes) else: print("No notification is needed, exit!") sys.exit(0) if dry_run: print("Finish writing email announcement in: " + work_dir) print("\nOnce you have checked the log file and generated emails") print("Use the command below to verify emails sending to test user:"******"\n hivemind notification.verify_mailout " + work_dir + " " + "SUBJECT" + " " + "[--mailto TEST_ADDRESS]" + " " + "[--smtp_server SMTP_SEVRVER]") print("\nThen rerun the command with --no-dry-run to mail ALL users") else: mailout(work_dir, data, subject, config) make_archive(work_dir)
def freshdesk_mailout(template, zone=None, ip=None, nodes=None, image=None, status="ALL", project=None, user=None, subject="Important announcement concerning your " "instance(s)", start_time=None, duration=None, timezone="AEDT", instances_file=None, dry_run=True, record_metadata=False, metadata_field="notification:fd_ticket", test_recipient=None): """Mailout announcements from freshdesk (Recommended). Freshdesk ticket per project will be created along with outbound email. Each mail will notify the first TenantManager and cc all other members. Once the customer responds to the mail, the reply will be appended to the ticket and status is changed to Open. Some files will be generated and written into ~/.cache/hivemind-mailout/freshdesk/<XXXX> in dry run mode, which are for operator check and no-dry-run use. They include: 1) notify.log: run log with all instances info and its email recipients; 2) notification@<project-name>: rendered emails content and recipients; 3) instances.list: all impacted instances id :param str template: template path to use for the mailout :param str zone: Availability zone affected by outage :param str ip: Only consider instances with specific ip addresses :param str nodes: Only target instances from the following Hosts/Nodes :param str image: Only consider instances with specific image :param str status: Only consider instances with status :param str subject: Custom email subject :param str start_time: Outage start time :param float duration: duration of outage in hours :param str timezone: Timezone :param str instances_file: Only consider instances listed in file :param boolean dry_run: by default print info only, use --no-dry-run\ for realsies :param boolean record_metadata: record the freshdesk ticket URL in\ the nova instance metadata :param str metadata_field: set the name of the freshdesk ticket URL\ metadata field in the nova instance." """ fd_config = security.get_freshdesk_config() fd = security.get_freshdesk_client(fd_config['domain'], fd_config['api_key']) nc = nova.client() _validate_paramters(start_time, duration, instances_file, template) start_time = datetime.datetime.strptime(start_time, '%H:%M %d-%m-%Y')\ if start_time else None end_time = start_time + datetime.timedelta(hours=int(duration))\ if (start_time and duration) else None work_dir = os.path.expanduser('~/.cache/hivemind-mailout/freshdesk/') if dry_run: # find the impacted instances and construct data if not instances_file: instances = nova.list_instances(zone=zone, nodes=nodes, ip=ip, project=project, user=user, status=status, image=image) else: inst = get_instances_from_file(nova.client(), instances_file) instances = nova.extract_servers_info(inst) # group by project data = populate_data(instances) if not data: print("No notification needed, exit!") sys.exit(0) affected = len(data) print("\n DRY RUN MODE - only generate notification emails: \n") # write to logs and generate emails if not os.path.isdir(work_dir): os.makedirs(work_dir) work_dir = tempfile.mkdtemp(dir=work_dir) print("Creating Outbox: " + work_dir) print('\nPlease export the environment variable for the no-dry-run: ' 'export %s=%s' % ('HIVEMIND_MAILOUT_FRESHDESK', work_dir)) # render email content generate_logs(work_dir, data) generate_notification_mails(subject, template, data, work_dir, start_time, end_time, timezone, zone, affected, nodes) else: work_dir = os.environ.get('HIVEMIND_MAILOUT_FRESHDESK') if not work_dir or not os.path.isdir(work_dir): print('Workdir environment variable is not found!') print('Please run the command without --no-dry-run and ' 'export environment variable as prompted!') sys.exit(0) email_files = [name for name in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, name)) and "notification@" in name] if test_recipient: print('You have specified the test_recipient option, ' 'all the emails will be sent to %s' % test_recipient) query = '\nYou are running notification script in no-dry-run mode'\ '\nIt will use previously generated emails under %s\n'\ 'Make sure the contents are all good before you do next step'\ '\nOne outbounding email will create a separate ticket. '\ 'Be cautious since it could generate massive tickets!!!\n'\ 'There are %s tickets to be created, it takes roughly %s min '\ 'to finish all the creation, still continue?' if not query_yes_no(query % (work_dir, len(email_files), len(email_files) / 60 + 1), default='no'): sys.exit(1) subjectfd = "[Nectar Notice] " + subject for email_file in email_files: print('\nCreating new Freshdesk ticket') with open(os.path.join(work_dir, email_file), 'rb') as f: email = yaml.load(f) addresses = email['Sendto'].split(',') toaddress = addresses.pop(0) if test_recipient: toaddress = test_recipient addresses = [test_recipient] ticket = fd.tickets.create_outbound_email( description=email['Body'], subject=subjectfd, email=toaddress, cc_emails=addresses, email_config_id=int(fd_config['email_config_id']), group_id=int(fd_config['group_id']), priority=2, status=5, # set ticket initial status is closed tags=['notification']) ticket_id = ticket.id # Use friendly domain name if using prod if fd.domain == 'dhdnectar.freshdesk.com': domain = 'support.ehelp.edu.au' else: domain = fd.domain ticket_url = 'https://{}/helpdesk/tickets/{}'\ .format(domain, ticket_id) print('Ticket #{} has been created: {}' .format(ticket_id, ticket_url)) if record_metadata: # Record the ticket URL in the server metadata for server in email[4]: nc.servers.set_meta(server['id'], {metadata_field: ticket_url}) # delay for freshdesk api rate limit consideration time.sleep(1) # make achive the outbox folder print('Make archive after the mailout for %s' % work_dir) make_archive(work_dir)
def generate_instance_info(instance_id, style=None): nc = nova.client() kc = keystone.client() gc = glance.client() try: instance = nc.servers.get(instance_id) except n_exc.NotFound: error("Instance {} not found".format(instance_id)) info = instance._info.copy() for network_label, address_list in instance.networks.items(): info['%s network' % network_label] = ', '.join(address_list) flavor = info.get('flavor', {}) flavor_id = flavor.get('id', '') try: info['flavor'] = '%s (%s)' % (nova.get_flavor(nc, flavor_id).name, flavor_id) except Exception: info['flavor'] = '%s (%s)' % ("Flavor not found", flavor_id) # Image image = info.get('image', {}) if image: image_id = image.get('id', '') try: img = gc.images.get(image_id) nectar_build = img.get('nectar_build', 'N/A') info['image'] = ('%s (%s, NeCTAR Build %s)' % (img.name, img.id, nectar_build)) except Exception: info['image'] = 'Image not found (%s)' % image_id else: # Booted from volume info['image'] = "Attempt to boot from volume - no image supplied" # Tenant project_id = info.get('tenant_id') if project_id: try: project = keystone.get_project(kc, project_id) info['project_id'] = '%s (%s)' % (project.name, project.id) except Exception: pass # User user_id = info.get('user_id') if user_id: try: user = keystone.get_user(kc, user_id) info['user_id'] = '%s (%s)' % (user.name, user.id) except Exception: pass # Remove stuff info.pop('links', None) info.pop('addresses', None) info.pop('hostId', None) info.pop('security_groups', None) return _format_instance(info, style=style)
def quota_reversions(infile=None, id_col=1, cores_col=2, instances_col=3, outfile=None): tenants = {}; if infile != None: with open(infile, 'rb') as csvfile: for row in csv.reader(csvfile, delimiter=','): try: tenants[row[id_col]] = [int(row[cores_col]), int(row[instances_col])] except: continue allocations = _get_current_allocations() nova_api = hm_nova.client() flavors = _get_flavor_map(nova_api) updates = [] for uuid in allocations.keys(): alloc = allocations[uuid] if len(tenants) > 0: # Figure out what the quota should have been prior to # the adjustment if uuid in tenants: deltas = tenants[uuid] expected = { 'core_quota': alloc['core_quota'] + deltas[0], 'instance_quota': alloc['instance_quota'] + deltas[1], 'ram_quota': (alloc['ram_quota'] + deltas[0] * 4) * 1024 } else: # No data for this tenant continue else: expected = None try: quotas = nova_api.quotas.get(uuid) except: continue alloc['expected'] = expected alloc['nova_quotas'] = quotas alloc['deltas'] = deltas updates.append(alloc) # If the alloc and current quotas match, don't touch them if (quotas.instances == alloc['instance_quota'] \ and quotas.ram == alloc['ram_quota'] * 1024 \ and quotas.cores == alloc['core_quota']): alloc['update'] = 'no - quotas match' continue usage = _get_usage(nova_api, flavors, uuid) alloc['nova_usage'] = usage # If the usage is greater than the allocated quotas, don't touch them if (usage['instances'] > alloc['instance_quota'] or usage['vcpus'] > alloc['core_quota'] or usage['ram'] > alloc['ram_quota'] * 1024): alloc['update'] = 'no - over-quota usage' continue # If the difference between the nova quotas don't match the # expected quotas (i.e. alloc + deltas), don't touch them if (expected != None and (expected['instance_quota'] != quotas.instances or expected['ram_quota'] != quotas.ram or expected['core_quota'] != quotas.cores)): alloc['update'] = 'no - deltas wrong' continue alloc['update'] = 'yes' fields_to_report = [ ("Tenant ID", lambda x: x['tenant_uuid']), ("Tenant Name", lambda x: x['tenant_name']), ("Instances", lambda x: x['instance_quota']), ("vCPU quota", lambda x: x['core_quota']), ("Memory", lambda x: x['ram_quota'] * 1024), ("Nova Instance quota", lambda x: x['nova_quotas'].instances), ("Nova vCPU quota", lambda x: x['nova_quotas'].cores), ("Nova Memory quota", lambda x: x['nova_quotas'].ram), ("Nova Instance usage", lambda x: x['nova_usage']['instances'] if 'nova_usage' in x else ''), ("Nova vCPU usage", lambda x: x['nova_usage']['vcpus'] if 'nova_usage' in x else ''), ("Nova Memory usage", lambda x: x['nova_usage']['ram'] if 'nova_usage' in x else ''), ("Instance delta", lambda x: x['deltas'][1] if 'deltas' in x else ''), ("vCPU delta", lambda x: x['deltas'][0] if 'deltas' in x else ''), ("Update", lambda x: x['update']) ] csv_output(map(lambda x: x[0], fields_to_report), map(lambda alloc: map( lambda y: y[1](alloc), fields_to_report), updates), filename=outfile)
def lock_instance(instance_id, dry_run=True): """pause and lock an instance""" if dry_run: print('Running in dry-run mode (use --no-dry-run for realsies)') fd_config = get_freshdesk_config() fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key']) nc = nova.client() kc = keystone.client() try: instance = nc.servers.get(instance_id) except n_exc.NotFound: error('Instance {} not found'.format(instance_id)) # Pause and lock instance if dry_run: if instance.status != 'ACTIVE': print('Instance state {}, will not pause'.format(instance.status)) else: print('Would pause and lock instance {}'.format(instance_id)) else: if instance.status != 'ACTIVE': print('Instance not in ACTIVE state ({}), skipping' .format(instance.status)) else: print('Pausing instance {}'.format(instance_id)) instance.pause() print('Locking instance {}'.format(instance_id)) instance.lock() # Process ticket ticket_id = None ticket_url = instance.metadata.get('security_ticket') if ticket_url: print('Found existing ticket: {}'.format(ticket_url)) ticket_id = int(ticket_url.split('/')[-1]) if dry_run: print('Would set ticket #{} status to open/urgent' .format(ticket_id)) else: # Set ticket status=waiting for customer, priority=urgent print('Setting ticket #{} status to open/urgent'.format(ticket_id)) fd.tickets.update_ticket(ticket_id, status=6, priority=4) else: project = keystone.get_project(kc, instance.tenant_id) user = keystone.get_user(kc, instance.user_id) email = user.email or '*****@*****.**' name = getattr(user, 'full_name', email) cc_emails = get_tenant_managers_emails(kc, instance) # Create ticket if none exist, and add instance info subject = 'Security incident for instance {} ({})'.format( instance.name, instance_id) body = '<br />\n'.join([ 'Dear Nectar Research Cloud User, ', '', '', 'We have reason to believe that cloud instance: ' '<b>{} ({})</b>'.format(instance.name, instance_id), 'in the project <b>{}</b>'.format(project.name), 'created by <b>{}</b>'.format(email), 'has been involved in a security incident, and has been locked.', '', 'We have opened this helpdesk ticket to track the details and ', 'the progress of the resolution of this issue.', '', 'Please reply to this email if you have any questions or ', 'concerns.', '', 'Thanks, ', 'Nectar Research Cloud Team' ]) if dry_run: print('Would create ticket with details:') print(' To: {} <{}>'.format(name, email)) print(' CC: {}'.format(', '.join(cc_emails))) print(' Subject: {}'.format(subject)) print('Would add instance details to ticket:') print(generate_instance_info(instance_id)) print(generate_instance_sg_rules_info(instance_id)) else: print('Creating new Freshdesk ticket') ticket = fd.tickets.create_outbound_email( name=name, description=body, subject=subject, email=email, cc_emails=cc_emails, email_config_id=int(fd_config['email_config_id']), group_id=int(fd_config['group_id']), priority=4, status=2, tags=['security']) ticket_id = ticket.id # Use friendly domain name if using prod if fd.domain == 'dhdnectar.freshdesk.com': domain = 'support.ehelp.edu.au' else: domain = fd.domain ticket_url = 'https://{}/helpdesk/tickets/{}'\ .format(domain, ticket_id) nc.servers.set_meta(instance_id, {'security_ticket': ticket_url}) print('Ticket #{} has been created: {}' .format(ticket_id, ticket_url)) # Add a private note with instance details print('Adding instance information to ticket') instance_info = generate_instance_info(instance_id, style='html') sg_info = generate_instance_sg_rules_info(instance_id, style='html') body = '<br/><br/>'.join([instance_info, sg_info]) fd.comments.create_note(ticket_id, body)