Example #1
0
def get_instance_usage_csv(start_date=None, end_date=None, filename=None):
    """Get instance usage statistics for all projects.
Date strings should be ISO 8601 to minute precision
without timezone information.

    """
    assert start_date and end_date
    start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
    end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
    keystone = hm_keystone.client()
    nova = hm_nova.client()

    tenants = {x.id: x for x in keystone.tenants.list()}
    headings = ["Tenant ID", "Tenant Name", "Instance count",
                "Instance hours", "vCPU hours", "Memory Hours (MB)",
                "Disk hours (GB)"]
    usage = map(lambda u: [
                u.tenant_id,
                tenants[u.tenant_id].name if u.tenant_id in tenants else None,
                len(u.server_usages),
                u.total_hours,
                u.total_vcpus_usage,
                u.total_memory_mb_usage,
                u.total_local_gb_usage],
                nova.usage.list(start, end, detailed=True))
    csv_output(headings, usage, filename=filename)
Example #2
0
def archive(image_id, dry_run=True, tenant=None, community=False):
    """Archive image by moving it to the <NECTAR_ARCHIVES> tenant.
    If the community flag is set
    please specify the community archive tenant id.
    """
    if dry_run:
        print("Running in dry run mode")

    if community:
        archive_tenant = get_community_archive_tenant(tenant)
    else:
        archive_tenant = get_archive_tenant(tenant)

    gc = get_glance_client(keystone.client())
    try:
        image = gc.images.get(image_id)
    except exc.HTTPNotFound:
        error("Image ID not found.")

    if dry_run:
        print("Would archive image {} ({}) to tenant {} ({})"
              .format(image.name, image.id,
                      archive_tenant.name, archive_tenant.id))
        if 'murano_image_info' in image.properties:
            print('Would remove murano image properties from {}'
                  .format(image.id))
    else:
        print("Archiving image {} ({}) to tenant {} ({})"
              .format(image.name, image.id,
                      archive_tenant.name, archive_tenant.id))
        change_tenant(image, archive_tenant)
        if 'murano_image_info' in image.properties:
            print('Removing murano image properties from {}'.format(image.id))
            remove_property(image, 'murano_image_info')
Example #3
0
def generate_instance_info(instance_id, style=None):
    nc = nova.client()
    kc = keystone.client()
    gc = glance.get_glance_client(kc)

    try:
        instance = nc.servers.get(instance_id)
    except n_exc.NotFound:
        error("Instance {} not found".format(instance_id))

    info = instance._info.copy()
    for network_label, address_list in instance.networks.items():
        info["%s network" % network_label] = ", ".join(address_list)

    flavor = info.get("flavor", {})
    flavor_id = flavor.get("id", "")

    try:
        info["flavor"] = "%s (%s)" % (nova.get_flavor(nc, flavor_id).name, flavor_id)
    except Exception:
        info["flavor"] = "%s (%s)" % ("Flavor not found", flavor_id)

    # Image
    image = info.get("image", {})
    if image:
        image_id = image.get("id", "")
        try:
            img = gc.images.get(image_id)
            nectar_build = img.properties.get("nectar_build", "N/A")
            info["image"] = "%s (%s, NeCTAR Build %s)" % (img.name, img.id, nectar_build)
        except Exception:
            info["image"] = "Image not found (%s)" % image_id
    else:  # Booted from volume
        info["image"] = "Attempt to boot from volume - no image supplied"

    # Tenant
    tenant_id = info.get("tenant_id")
    if tenant_id:
        try:
            tenant = keystone.get_tenant(kc, tenant_id)
            info["tenant_id"] = "%s (%s)" % (tenant.name, tenant.id)
        except Exception:
            pass

    # User
    user_id = info.get("user_id")
    if user_id:
        try:
            user = keystone.get_user(kc, user_id)
            info["user_id"] = "%s (%s)" % (user.name, user.id)
        except Exception:
            pass

    # Remove stuff
    info.pop("links", None)
    info.pop("addresses", None)
    info.pop("hostId", None)
    info.pop("security_groups", None)

    return _format_instance(info, style=style)
Example #4
0
def allocation_homes(csv=False, filename=None, sslwarnings=False):
    """Get the allocation_homes for all projects. If this
metadata field is not set in keystone (see keystone hivemind
commands), the value reported is the email domains for all
tenant managers belonging to this project.
    """
    ssl_warnings(enabled=sslwarnings)
    keystone = hm_keystone.client()
    all_users = map(lambda x: x.to_dict(), keystone.users.list())
    email_dict = {x['id']: x['email'].split("@")[-1] for x in all_users
                  if 'email' in x and x['email'] is not None}
    projects = keystone.projects.list()
    managers = collections.defaultdict(list)
    for user_role in keystone.role_assignments.list(role=14):
        if 'project' in user_role.scope:
            managers[user_role.scope['project']['id']].append(
                user_role.user['id'])
    headings = ["Tenant ID", "Allocation Home(s)"]
    records = []
    for proj in projects:
        if "allocation_home" in proj.to_dict():
            records.append([proj.id, proj.allocation_home])
        else:
            if len(managers[proj.id]) == 0:
                continue
            institutions = set()
            for tm in managers[proj.id]:
                if tm in email_dict:
                    institutions.add(email_dict[tm])
            records.append([proj.id, ",".join(institutions)])
    if csv:
        csv_output(headings, records, filename=filename)
    else:
        pretty_output(headings, records, filename=filename)
Example #5
0
def allocation_managers(csv=False, filename=None, sslwarnings=False):
    """Get the allocation manager emails for all projects.
    """
    ssl_warnings(enabled=sslwarnings)
    keystone = hm_keystone.client()
    all_users = map(lambda x: x.to_dict(), keystone.users.list())
    email_dict = {x['id']: x['email'] for x in all_users
                  if 'email' in x and x['email'] is not None}
    projects = keystone.projects.list()
    managers = collections.defaultdict(list)
    for user_role in keystone.role_assignments.list(role=14):
        if 'project' in user_role.scope:
            managers[user_role.scope['project']['id']].append(
                user_role.user['id'])
    headings = ["Tenant ID", "Manager email(s)"]
    records = []
    for proj in projects:
        if len(managers[proj.id]) == 0:
            continue
        emails = set()
        for tm in managers[proj.id]:
            if tm in email_dict:
                emails.add(email_dict[tm])
        records.append([proj.id, ",".join(emails)])
    if csv:
        csv_output(headings, records, filename=filename)
    else:
        pretty_output(headings, records, filename=filename)
Example #6
0
def promote(image_id, dry_run=True, tenant=None, community=False):
    """If the supplied image has nectar_name and nectar_build metadata, set
    to public. If there is an image with matching nectar_name and lower
    nectar_build, move that image to the <NECTAR_ARCHIVES> tenant.
    If the community flag is set please specify the community tenant id.
    """
    if dry_run:
        print("Running in dry run mode")

    if community:
        archive_tenant = get_community_tenant(tenant)
    else:
        archive_tenant = get_archive_tenant(tenant)

    images = get_glance_client(keystone.client()).images
    try:
        image = images.get(image_id)
    except exc.HTTPNotFound:
        error("Image ID not found.")
    if not community:
        try:
            name = image.properties['nectar_name']
            build = (int(image.properties['nectar_build']))
        except KeyError:
            error("nectar_name or nectar_build not found for image.")

        m_check = partial(match, name, build)
        matchingimages = filter(m_check, images.findall(owner=image.owner))
    else:
        matchingimages = [image]

    for i in matchingimages:
        if dry_run:
            print("Would change ownership of image {} ({}) to tenant {} ({})"
                  .format(i.name, i.id,
                          archive_tenant.name, archive_tenant.id))
            if 'murano_image_info' in i.properties:
                print('Would remove murano image properties from {}'
                      .format(i.id))
        else:
            change_tenant(i, archive_tenant)
            print("Changing ownership of image {} ({}) to tenant {} ({})"
                  .format(i.name, i.id,
                          archive_tenant.name, archive_tenant.id))
            if 'murano_image_info' in i.properties:
                print('Removing murano image properties from {}'
                      .format(i.id))
                remove_property(i, 'murano_image_info')

    if image.is_public:
        print("Image {} ({}) already set public"
              .format(image.name, image.id))
    else:
        if dry_run:
            print("Would set image {} ({}) to public"
                  .format(image.name, image.id))
        else:
            print("Setting image {} ({}) to public"
                  .format(image.name, image.id))
            image.update(is_public=True)
Example #7
0
def public_audit():
    """Print usage information about all public images
    """
    gc = get_glance_client(keystone.client(), api_version=2)
    nc = nova.client()
    db = nova.db_connect()

    # The visibility filter doesn't seem to work... so we filter them out again
    images = gc.images.list(visibility='public')
    public = [i for i in images if i['visibility'] == 'public']

    table = PrettyTable(["ID", "Name", "Num running instances",
                         "Boot count", "Last Boot"])

    for i in public:
        sql = select([nova.instances_table])
        where = [nova.instances_table.c.image_ref.like(i['id'])]
        sql = sql.where(*where).order_by(desc('created_at'))
        image_instances = db.execute(sql).fetchall()
        boot_count = len(image_instances)
        if boot_count > 0:
            last_boot = image_instances[0].created_at
        else:
            last_boot = 'Never'
        instances = nova.all_servers(nc, image=i['id'])

        table.add_row([i['id'], i['name'],
                       len(instances), boot_count, last_boot])

    print(table.get_string(sortby="Num running instances", reversesort=True))
Example #8
0
def revoke_access(user):
    """Revoke access to reports.rc.nectar.org.au for the user.
    """
    ksclient = keystone.client()
    user = keystone.get_user(ksclient, user)
    keystone.remove_project_roles(user.default_project_id, user.id, [ROLE])
    keystone.print_members(ksclient, user.default_project_id)
Example #9
0
def grant_access(user):
    """Grant the user access to reports.rc.nectar.org.au.
    """
    ksclient = keystone.client()
    user = keystone.get_user(ksclient, user)
    keystone.add_project_roles(user.default_project_id, user.id, [ROLE])
    keystone.print_members(ksclient, user.default_project_id)
def _populate_project_dict(instances):
    session = keystone.get_session()
    ksclient = keystone.client(session=session)
    project = collections.defaultdict(dict)
    for instance in instances:
        if instance['project_name'] in project.keys():
            project[instance['project_name']]['instances'].append(instance)
        else:
            cclist = []
            for role in ['TenantManager', 'Member']:
                members = keystone.list_members(instance['project'], role)
                if not members:
                    continue
                for uid in members.keys():
                    user = keystone.get_user(ksclient, uid, use_cache=True)
                    if getattr(user, 'enabled', None) and \
                       getattr(user, 'email', None):
                        cclist.append(user.email)
            # rule out the projects where has no valid recipients(tempest. etc)
            if cclist:
                project[instance['project_name']] = {'instances': [instance]}
                project[instance['project_name']].update(
                    {'recipients': cclist})

    return project
def link_account(existing_email, new_email):
    db = connect()
    ids = keystone_ids_from_email(db, existing_email)
    new_email_ids = keystone_ids_from_email(db, new_email)

    if len(ids) != 1:
        print('User has multiple accounts with email %s' % existing_email)
        return

    user_id = ids[0]
    orphan_user_id = new_email_ids[0]

    print('%s: %s' % (existing_email, user_id))
    print('%s: %s' % (new_email, orphan_user_id))

    if user_id == orphan_user_id:
        print('Those accounts are already linked')
        return

    client = keystone.client()
    user = client.users.get(orphan_user_id)

    project = client.projects.get(user.default_project_id)
    servers = nova.client().servers.list(search_opts={
        'all_tenants': True, 'project_id': project.id})

    if len(servers):
        print('Soon to be orphaned project has active instances.')
        print('Advise user to terminate them.')
        return

    print()
    print('Confirm that you want to:')
    print(' - Link %s to account %s' % (new_email, existing_email))
    print(' - Disable orphan Keystone project %s' % (project.name))
    print(' - Disable orphan Keystone user %s' % (user.name))
    print()

    response = raw_input('(yes/no): ')
    if response != 'yes':
        return

    print('Linking account.')
    sql = (update(users)
           .where(users.c.email == new_email)
           .values(user_id=user_id))
    result = db.execute(sql)

    if result.rowcount == 0:
        print('Something went wrong.')
        return

    print('Disabling orphaned Keystone project %s (%s).' % (
        project.name, project.id))
    client.projects.update(project.id, enabled=False)
    print('Disabling orphaned Keystone user %s (%s).' % (user.name, user.id))
    client.users.update(user.id, enabled=False, name="%s-disabled" % user.name)
    print('All done.')
Example #12
0
def link_account(existing_email, new_email):
    db = connect()
    ids = keystone_ids_from_email(db, existing_email)
    new_email_ids = keystone_ids_from_email(db, new_email)

    if len(ids) != 1:
        print 'User has multiple accounts with email %s' % existing_email
        return

    user_id = ids[0]
    orphan_user_id = new_email_ids[0]

    print '%s: %s' % (existing_email, user_id)
    print '%s: %s' % (new_email, orphan_user_id)

    if user_id == orphan_user_id:
        print 'Those accounts are already linked'
        return

    client = keystone.client()
    user = client.users.get(orphan_user_id)
    project = client.tenants.get(user.tenantId)
    servers = nova.client().servers.list(search_opts={
        'all_tenants': True, 'project_id': project.id})

    if len(servers):
        print 'Soon to be orphaned project has active instances.'
        print 'Advise user to terminate them.'
        return

    print
    print 'Confirm that you want to:'
    print ' - Link %s to account %s' % (new_email, existing_email)
    print ' - Delete orphan Keystone project %s' % (project.name)
    print ' - Delete orphan Keystone user %s' % (user.name)
    print

    response = raw_input('(yes/no): ')
    if response != 'yes':
        return

    print 'Linking account.'
    sql = (update(users)
           .where(users.c.email == new_email)
           .values(user_id=user_id))
    result = db.execute(sql)

    if result.rowcount == 0:
        print 'Something went wrong.'
        return

    print 'Deleting orphaned Keystone project %s (%s).' % (
        project.name, project.id)
    client.tenants.delete(project.id)
    print 'Deleting orphaned Keystone user %s (%s).' % (user.name, user.id)
    client.users.delete(user.id)
    print 'All done.'
def link_account(existing_email, new_email):
    db = connect()
    ids = keystone_ids_from_email(db, existing_email)
    new_email_ids = keystone_ids_from_email(db, new_email)

    if len(ids) != 1:
        print("User has multiple accounts with email %s" % existing_email)
        return

    user_id = ids[0]
    orphan_user_id = new_email_ids[0]

    print("%s: %s" % (existing_email, user_id))
    print("%s: %s" % (new_email, orphan_user_id))

    if user_id == orphan_user_id:
        print("Those accounts are already linked")
        return

    client = keystone.client()
    user = client.users.get(orphan_user_id)
    project = client.tenants.get(user.tenantId)
    servers = nova.client().servers.list(search_opts={"all_tenants": True, "project_id": project.id})

    if len(servers):
        print("Soon to be orphaned project has active instances.")
        print("Advise user to terminate them.")
        return

    print()
    print("Confirm that you want to:")
    print(" - Link %s to account %s" % (new_email, existing_email))
    print(" - Delete orphan Keystone project %s" % (project.name))
    print(" - Delete orphan Keystone user %s" % (user.name))
    print()

    response = raw_input("(yes/no): ")
    if response != "yes":
        return

    print("Linking account.")
    sql = update(users).where(users.c.email == new_email).values(user_id=user_id)
    result = db.execute(sql)

    if result.rowcount == 0:
        print("Something went wrong.")
        return

    print("Deleting orphaned Keystone project %s (%s)." % (project.name, project.id))
    client.tenants.delete(project.id)
    print("Deleting orphaned Keystone user %s (%s)." % (user.name, user.id))
    client.users.delete(user.id)
    print("All done.")
Example #14
0
def _search_trove_instances(client, opts):
    # keep the proj/user from searching opts
    proj_id = opts.get('tenant_id', None)
    user_id = opts.get('user_id', None)

    # trove instances will be launched by global trove project
    trove_opts = opts.copy()
    trove_opts.pop('user_id', None)
    trove_opts['tenant_id'] = keystone.get_project(
        keystone.client(), 'trove', use_cache=True).id
    trove_instances = client.servers.list(search_opts=trove_opts)
    trove_instances = [instance for instance in trove_instances
                       if _match_proj_user(instance, proj_id, user_id)]
    return trove_instances
Example #15
0
def list_instances(zone=None, nodes=None, project=None, user=None,
                   status="ACTIVE", ip=None, image=None, limit=None,
                   changes_since=None, scenario=None):
    """Prints a pretty table of instances based on specific conditions

       :param str zone: Availability zone or availability zone range
            that the instances are in, e.g. az[1-4,9]
       :param str nodes: Compute host name or host neme range that the
            instances are in, e.g. cc[2-3,5]
       :param str project: Project name or id that the instances belong to
       :param str user: User name or id that the instances belong to
       :param str status: Instances status. Use 'ALL' to list all instances
       :param str ip: Ip address or ip address range that instances are in,
            e.g. 192.168.122.[124-127]
       :param str image: Image id that the instances are launched based on
       :param str limit: Number of returned instances
       :param str changes_since: List only instances changed after a certain
            point of time. The provided time should be an ISO 8061 formatted
            time. e.g. 2016-03-04T06:27:59Z
       :param str scenario: List only instances which match with specific
            scenario checking, available ones are ["compute_failure"]
    """
    novaclient = client()
    if status == 'ALL':
        status = None
    result = all_servers(novaclient, zone=zone, host=nodes, status=status,
                         ip=ip, image=image, project=project, user=user,
                         limit=limit, changes_since=changes_since)
    if not result:
        print("No instances found!")
        sys.exit(0)
    result = extract_servers_info(result, keystone.client())

    if scenario:
        func = globals()["_scenario_" + scenario]
        result = match_scenario(result, func, novaclient, changes_since)
        if not result:
            print("No %s instances found!" % scenario)
            sys.exit(0)

    print("\n")
    header = None
    for inst in result:
        if not header:
            header = inst.keys()
            table = PrettyTable(header)
        table.add_row(inst.values())
    print(table)
    print("number of instances:", len(result))
    return result
Example #16
0
def get_ticket_recipients(instance):
    """Build a list of email addresses"""
    email_addresses = []

    kc = keystone.client()
    user = keystone.get_user(kc, instance.user_id)
    if user.email:
        email_addresses.append(user.email)

    # Add tenant members to ticket recipient list
    tenant = keystone.get_tenant(kc, instance.tenant_id)
    for user in tenant.list_users():
        roles = [r.name for r in user.list_roles(tenant)]
        if 'TenantManager' in roles:
            email_addresses.append(user.email)
    return email_addresses
Example #17
0
def purge_project(tenant_name, dry_run=True):
    """Purge resources and disable a given project """
    if not spawn.find_executable('ospurge'):
        error('ospurge not found in path. Please ensure it is installed.')
    username, password = get_creds()

    if dry_run:
        print("Running in dry-run mode")

    ks_client = keystone.client()
    tenant = keystone.get_tenant(ks_client, tenant_name)
    tenant_member_role = ks_client.roles.find(name='Member')
    ospurge_user = keystone.get_user(ks_client, username)

    if not tenant.enabled:
        print("Enabling project for purge")
        tenant.update(enabled=True)

    if not is_in_project(tenant, ospurge_user):
        print("Adding ospurge user to project")
        tenant.add_user(ospurge_user, tenant_member_role)

    if dry_run:
        run_opt = '--dry-run'
    else:
        run_opt = '--verbose'

    cmd = ("ospurge --dont-delete-project --own-project --username {username} "
           "--password {password} --admin-project {tenant} {run_opt}"
           "".format(username=username, password=password,
                     tenant=tenant.name, run_opt=run_opt))
    print("Running: {}".format(cmd.replace(password, 'xxxx')))
    run(cmd)

    if is_in_project(tenant, ospurge_user):
        print("Removing ospurge user from project")
        tenant.remove_user(ospurge_user, tenant_member_role)

    if tenant.enabled:
        if dry_run and tenant.enabled:
            print("Not disabling project due to dry-run")
        else:
            print("Disabling project")
            tenant.update(enabled=False)
            keystone.set_project_metadata(tenant_name,
                                          'ospurge_date',
                                          str(datetime.datetime.now()))
Example #18
0
def get_images_tenant(tenant_id_or_name, tenant_type):
    """fetch tenant id from config file"""
    if tenant_id_or_name is None:
        msg = " ".join(("No tenant set.", "Please set tenant in",
                        "[cfg:hivemind_contrib.glance.%s]" % tenant_type))
        error(msg)
    try:
        ks_client = keystone.client()
        tenant = keystone.get_tenant(ks_client, tenant_id_or_name)
    except ks_exc.NotFound:
        raise error("Tenant {} not found. Check your settings."
                    .format(tenant_id_or_name))
    except ks_exc.Forbidden:
        raise error("Permission denied. Check you're using admin credentials.")
    except Exception as e:
        raise error(e)

    return tenant
Example #19
0
def create_zone(zone_name, project_id_or_name, dry_run=True):
    """Create a designate zone for a user"""

    # Designate requires zone names to end with a dot.
    if not zone_name.endswith('.'):
        zone_name = "%s." % zone_name

    d_client = client()
    try:
        ks_client = keystone.client()
        project = keystone.get_project(ks_client, project_id_or_name)
    except ks_exc.NotFound:
        raise error("Project {} not found. Check your settings."
                    .format(project_id_or_name))
    except ks_exc.Forbidden:
        raise error("Permission denied getting project {} details."
                    .format(project_id_or_name))

    if dry_run:
        print("Would create designate zone {}".format(zone_name))
        print("Would transfer zone {} to project {}".format(
            zone_name, project.id))
    else:
        d_client.session.sudo_project_id = None
        zone = d_client.zones.create(zone_name,
                                     email='*****@*****.**')
        print("Created new zone {}".format(zone['name']))
        print("Transferring zone {} to project {}".format(
            zone['name'], project.id))

        create_req = d_client.zone_transfers.create_request(
            zone_name, project.id)

        d_client.session.sudo_project_id = project.id
        accept_req = d_client.zone_transfers.accept_request(
            create_req['id'], create_req['key'])
        if accept_req['status'] == 'COMPLETE':
            print("Zone {} transfer to project {} is complete".format(
                zone['name'], project.id))
        return zone
Example #20
0
def lock_instance(instance_id, dry_run=True):
    """pause and lock an instance"""
    if dry_run:
        print('Running in dry-run mode (use --no-dry-run for realsies)')

    fd = get_freshdesk_client()
    nc = nova.client()
    kc = keystone.client()
    try:
        instance = nc.servers.get(instance_id)
    except n_exc.NotFound:
        error('Instance {} not found'.format(instance_id))

    ticket_id = None
    ticket_url = instance.metadata.get('security_ticket')
    if ticket_url:
        print('Found existing ticket: {}'.format(ticket_url))
        ticket_id = int(ticket_url.split('/')[-1])

        if dry_run:
            print('Would set ticket #{} status to open/urgent'
                  .format(ticket_id))
        else:
            # Set ticket status=waiting for customer, priority=urgent
            print('Setting ticket #{} status to open/urgent'.format(ticket_id))
            fd.tickets.update_ticket(ticket_id, status=6, priority=4)
    else:
        tenant = keystone.get_tenant(kc, instance.tenant_id)
        user = keystone.get_user(kc, instance.user_id)
        email_addresses = get_ticket_recipients(instance)

        # Create ticket if none exist, and add instance info
        subject = 'Security incident for instance {}'.format(instance_id)
        body = '<br />\n'.join([
            'Dear NeCTAR Research Cloud User, ',
            '',
            '',
            'We have reason to believe that cloud instance: '
            '<b>{} ({})</b>'.format(instance.name, instance.id),
            'in the project <b>{}</b>'.format(tenant.name),
            'created by <b>{}</b>'.format(user.email),
            'has been involved in a security incident.',
            '',
            'We have opened this helpdesk ticket to track the details and ',
            'the progress of the resolution of this issue.',
            '',
            'Please reply to this email if you have any questions or ',
            'concerns.',
            '',
            'Thanks, ',
            'NeCTAR Research Cloud Team'
        ])

        if dry_run:
            print('Would create ticket with details:')
            print('  To:      {}'.format(email_addresses))
            print('  Subject: {}'.format(subject))

            print('Would add instance details to ticket:')
            print(generate_instance_info(instance_id))
            print(generate_instance_sg_rules_info(instance_id))
        else:
            print('Creating new Freshdesk ticket')
            ticket = fd.tickets.create_ticket(
                description=body,
                subject=subject,
                email='*****@*****.**',
                cc_emails=email_addresses,
                priority=4,
                status=6,
                tags=['security'])
            ticket_id = ticket.id
            ticket_url = 'https://{}/helpdesk/tickets/{}'\
                         .format(fd.domain, ticket_id)
            nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
            print('Ticket #{} has been created: {}'
                  .format(ticket_id, ticket_url))

            # Add a private note with instance details
            print('Adding instance information to ticket')
            instance_info = generate_instance_info(instance_id, style='html')
            sg_info = generate_instance_sg_rules_info(instance_id,
                                                      style='html')
            body = '<br/><br/>'.join([instance_info, sg_info])
            fd.comments.create_note(ticket_id, body)

    if dry_run:
        if instance.status != 'ACTIVE':
            print('Instance state {}, will not pause'.format(instance.status))
        else:
            print('Would pause and lock instance {}'.format(instance_id))

        print('Would update ticket with action')
    else:
        # Pause and lock
        if instance.status != 'ACTIVE':
            print('Instance not in ACTIVE state ({}), skipping'
                  .format(instance.status))
        else:
            print('Pausing instance {}'.format(instance_id))
            instance.pause()

        print('Locking instance {}'.format(instance_id))
        instance.lock()

        # Add reply to user
        email_addresses = get_ticket_recipients(instance)
        print('Replying to ticket with action details')
        action = 'Instance <b>{} ({})</b> has been <b>paused and locked</b> '\
                 'pending further investigation'\
                 .format(instance.name, instance_id)
        fd.comments.create_reply(ticket_id, action, cc_emails=email_addresses)
Example #21
0
def all_servers(client, zone=None, host=None, status=None, ip=None,
                image=None, project=None, user=None, limit=None,
                changes_since=None):
    print("\nListing the instances... ", end="")
    marker = None
    opts = {}
    opts["all_tenants"] = True
    if status:
        opts['status'] = status
    if limit:
        opts['limit'] = limit
    if image:
        opts['image'] = image
    if changes_since:
        opts['changes-since'] = changes_since
    if project:
        try:
            opts['tenant_id'] = keystone.get_project(
                keystone.client(), project, use_cache=True).id
        except Exception:
            sys.exit(1)
    if user:
        try:
            opts['user_id'] = keystone.get_user(
                keystone.client(), user, use_cache=True).id
        except Exception:
            sys.exit(1)

    host_list = parse_nodes(host) if host else None
    az_list = parse_nodes(zone) if zone else None
    ip_list = parse_nodes(ip) if ip else None

    # When using all the searching opts other than project or user,
    # trove instances will be returned by default via nova list api.
    # But they will not when search_opts contain project or user.
    # In order to include them, searching all the instances under
    # project "trove" and filtering them by the instance metadata.
    if project or user:
        inst = _search_trove_instances(client, opts)
    else:
        inst = []

    if host_list:
        for host in host_list:
            opts['host'] = host
            instances = client.servers.list(search_opts=opts)
            if not instances:
                continue
            instances = filter(lambda x: _match_availability_zone(x, az_list),
                               instances)
            instances = filter(lambda x: _match_ip_address(x, ip_list),
                               instances)
            inst.extend(instances)
            if limit and len(inst) >= int(limit):
                break
        return inst
    else:
        while True:
            if marker:
                opts['marker'] = marker
            instances = client.servers.list(search_opts=opts)
            if not instances:
                return inst
            # for some instances stuck in build phase, servers.list api
            # will always return the marker instance. Add old marker and
            # new marker comparision to avoid the dead loop
            marker_new = instances[-1].id
            if marker == marker_new:
                return inst
            marker = marker_new
            instances = filter(lambda x: _match_availability_zone(x, az_list),
                               instances)
            instances = filter(lambda x: _match_ip_address(x, ip_list),
                               instances)
            if not instances:
                continue
            inst.extend(instances)
            if limit and len(inst) >= int(limit):
                return inst
Example #22
0
def extract_servers_info(servers, ksclient=None):
    print("\nExtracting instances information... ", end="")
    if ksclient is None:
        ksclient = keystone.client()
    return [extract_server_info(server, ksclient=ksclient)
            for server in servers]
Example #23
0
def get_instance_usage_csv(start_date, end_date, tenant=None,
                           filename=None, sslwarnings=False):
    """Get individual instance usage for all projects, including tenant and
    availability zones.  Date strings should be ISO 8601 to minute precision
    without timezone information.
    """
    ssl_warnings(enabled=sslwarnings)
    start = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M")
    end = datetime.datetime.strptime(end_date, "%Y-%m-%dT%H:%M")
    keystone = hm_keystone.client()
    nova = hm_nova.client()

    if tenant:
        tenants = {tenant: keystone.projects.get(tenant)}
        raw_usage = [nova.usage.get(tenant, start, end)]
    else:
        tenants = {x.id: x for x in keystone.projects.list()}
        raw_usage = nova.usage.list(start, end, detailed=True)

    usage = []
    for u in raw_usage:
        tenant_id = u.tenant_id
        tenant_name = tenants[tenant_id].name if tenant_id in tenants else None

        # The Nova API doesn't allow "show" on deleted instances, but
        # we can get the info using "list --deleted".  The problem is
        # figuring out how to avoid retrieving irrelevant instances,
        # and at the same time how to avoid too many requests.
        #
        # Attempt #1 - use the tenant_id and the instance's name to
        # focus queries.
        # Attempt #2 - as #1, but after N lookups by name for a tenant,
        # just fetch all of the deleted instances.
        cache = {}
        try:
            for iu in u.server_usages:
                name = iu['name']
                instance_id = iu['instance_id']
                instance = None
                if iu['state'] == 'terminated' or iu['state'] == 'deleted':
                    instance = _get_deleted_instance(cache, nova, u.tenant_id,
                                                     name, instance_id)
                else:
                    try:
                        instance = nova.servers.get(instance_id).to_dict()
                    except Exception:
                        print('Cannot find instance {0} in {1}'
                              .format(instance_id, u.tenant_id))
                if instance is None:
                    instance = {'OS-EXT-AZ:availability_zone': 'unknown'}

                usage.append([tenant_id, tenant_name, instance_id, name,
                              iu['state'], iu['flavor'], iu['hours'],
                              iu['vcpus'], iu['memory_mb'], iu['local_gb'],
                              instance['OS-EXT-AZ:availability_zone']])
        except Exception:
            traceback.print_exc(file=sys.stdout)

    headings = ["Tenant ID", "Tenant Name", "Instance id", "Instance name",
                "Instance state", "Flavour",
                "Instance hours", "vCPUs", "Memory (MB)", "Disk (GB)", "AZ"]
    csv_output(headings, usage, filename=filename)
Example #24
0
def generate_instance_info(instance_id, style=None):
    nc = nova.client()
    kc = keystone.client()
    gc = glance.client()

    try:
        instance = nc.servers.get(instance_id)
    except n_exc.NotFound:
        error("Instance {} not found".format(instance_id))

    info = instance._info.copy()
    for network_label, address_list in instance.networks.items():
        info['%s network' % network_label] = ', '.join(address_list)

    flavor = info.get('flavor', {})
    flavor_id = flavor.get('id', '')

    try:
        info['flavor'] = '%s (%s)' % (nova.get_flavor(nc, flavor_id).name,
                                      flavor_id)
    except Exception:
        info['flavor'] = '%s (%s)' % ("Flavor not found", flavor_id)

    # Image
    image = info.get('image', {})
    if image:
        image_id = image.get('id', '')
        try:
            img = gc.images.get(image_id)
            nectar_build = img.get('nectar_build', 'N/A')
            info['image'] = ('%s (%s, NeCTAR Build %s)'
                             % (img.name, img.id, nectar_build))
        except Exception:
            info['image'] = 'Image not found (%s)' % image_id

    else:  # Booted from volume
        info['image'] = "Attempt to boot from volume - no image supplied"

    # Tenant
    project_id = info.get('tenant_id')
    if project_id:
        try:
            project = keystone.get_project(kc, project_id)
            info['project_id'] = '%s (%s)' % (project.name, project.id)
        except Exception:
            pass

    # User
    user_id = info.get('user_id')
    if user_id:
        try:
            user = keystone.get_user(kc, user_id)
            info['user_id'] = '%s (%s)' % (user.name, user.id)
        except Exception:
            pass

    # Remove stuff
    info.pop('links', None)
    info.pop('addresses', None)
    info.pop('hostId', None)
    info.pop('security_groups', None)

    return _format_instance(info, style=style)
Example #25
0
def lock_instance(instance_id, dry_run=True):
    """pause and lock an instance"""
    if dry_run:
        print('Running in dry-run mode (use --no-dry-run for realsies)')

    fd_config = get_freshdesk_config()
    fd = get_freshdesk_client(fd_config['domain'], fd_config['api_key'])
    nc = nova.client()
    kc = keystone.client()
    try:
        instance = nc.servers.get(instance_id)
    except n_exc.NotFound:
        error('Instance {} not found'.format(instance_id))

    # Pause and lock instance
    if dry_run:
        if instance.status != 'ACTIVE':
            print('Instance state {}, will not pause'.format(instance.status))
        else:
            print('Would pause and lock instance {}'.format(instance_id))
    else:
        if instance.status != 'ACTIVE':
            print('Instance not in ACTIVE state ({}), skipping'
                  .format(instance.status))
        else:
            print('Pausing instance {}'.format(instance_id))
            instance.pause()

        print('Locking instance {}'.format(instance_id))
        instance.lock()

    # Process ticket
    ticket_id = None
    ticket_url = instance.metadata.get('security_ticket')
    if ticket_url:
        print('Found existing ticket: {}'.format(ticket_url))
        ticket_id = int(ticket_url.split('/')[-1])

        if dry_run:
            print('Would set ticket #{} status to open/urgent'
                  .format(ticket_id))
        else:
            # Set ticket status=waiting for customer, priority=urgent
            print('Setting ticket #{} status to open/urgent'.format(ticket_id))
            fd.tickets.update_ticket(ticket_id, status=6, priority=4)
    else:
        project = keystone.get_project(kc, instance.tenant_id)
        user = keystone.get_user(kc, instance.user_id)
        email = user.email or '*****@*****.**'
        name = getattr(user, 'full_name', email)
        cc_emails = get_tenant_managers_emails(kc, instance)

        # Create ticket if none exist, and add instance info
        subject = 'Security incident for instance {} ({})'.format(
            instance.name, instance_id)
        body = '<br />\n'.join([
            'Dear Nectar Research Cloud User, ',
            '',
            '',
            'We have reason to believe that cloud instance: '
            '<b>{} ({})</b>'.format(instance.name, instance_id),
            'in the project <b>{}</b>'.format(project.name),
            'created by <b>{}</b>'.format(email),
            'has been involved in a security incident, and has been locked.',
            '',
            'We have opened this helpdesk ticket to track the details and ',
            'the progress of the resolution of this issue.',
            '',
            'Please reply to this email if you have any questions or ',
            'concerns.',
            '',
            'Thanks, ',
            'Nectar Research Cloud Team'
        ])

        if dry_run:
            print('Would create ticket with details:')
            print('  To:      {} <{}>'.format(name, email))
            print('  CC:      {}'.format(', '.join(cc_emails)))
            print('  Subject: {}'.format(subject))

            print('Would add instance details to ticket:')
            print(generate_instance_info(instance_id))
            print(generate_instance_sg_rules_info(instance_id))
        else:
            print('Creating new Freshdesk ticket')
            ticket = fd.tickets.create_outbound_email(
                name=name,
                description=body,
                subject=subject,
                email=email,
                cc_emails=cc_emails,
                email_config_id=int(fd_config['email_config_id']),
                group_id=int(fd_config['group_id']),
                priority=4,
                status=2,
                tags=['security'])
            ticket_id = ticket.id

            # Use friendly domain name if using prod
            if fd.domain == 'dhdnectar.freshdesk.com':
                domain = 'support.ehelp.edu.au'
            else:
                domain = fd.domain

            ticket_url = 'https://{}/helpdesk/tickets/{}'\
                         .format(domain, ticket_id)
            nc.servers.set_meta(instance_id, {'security_ticket': ticket_url})
            print('Ticket #{} has been created: {}'
                  .format(ticket_id, ticket_url))

            # Add a private note with instance details
            print('Adding instance information to ticket')
            instance_info = generate_instance_info(instance_id, style='html')
            sg_info = generate_instance_sg_rules_info(instance_id,
                                                      style='html')
            body = '<br/><br/>'.join([instance_info, sg_info])
            fd.comments.create_note(ticket_id, body)