Ejemplo n.º 1
0
def get_hostgroups(classifier, observer_pass, regions):
    hostgroups = {name: [] for name in classifier.values()}

    for region in regions:
        client = novaclient.Client(
            "2.0",
            session=KeystoneSession(auth=KeystonePassword(
                auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
                username="******",
                password=observer_pass,
                project_name="tools",
                user_domain_name="default",
                project_domain_name="default",
            )),
            region_name=region,
        )
        for instance in client.servers.list():
            name = instance.name
            if name.startswith("tools-puppetmaster"):
                # To avoid chicken/egg strangeness, the tools puppetmaster
                #  is not itself managed by the tools puppetmaster.  That
                #  means clush keys aren't set up there.
                continue
            for prefix in classifier:
                if name.startswith("tools-" + prefix):
                    role = classifier[prefix]
                    hostgroups[role].append(name + ".tools.eqiad.wmflabs")

    return hostgroups
def get_keystone_session(project):
    return KeystoneSession(auth=KeystonePassword(
        auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
        username="******",
        password=open('novaobserver_password').read(),
        project_name=project,
        user_domain_name='default',
        project_domain_name='default'))
Ejemplo n.º 3
0
def get_projects_with_nfs(mounts_config, observer_pass, auth_url):
    """
    Get populated project objects that need NFS exports
    :param mounts_config: dict
    :returns: list
    """
    projects = []

    # Special one-off session just to grab the list of regions
    session = KeystoneSession(auth=KeystonePassword(
        auth_url=auth_url,
        username="******",
        password=observer_pass,
        project_name="observer",
        user_domain_name="default",
        project_domain_name="default",
    ))
    keystoneclient = keystone_client.Client(session=session,
                                            interface="public")
    region_recs = keystoneclient.regions.list()
    regions = [region.id for region in region_recs]

    server_vols = mounts_config["volumes_served"]
    for name, config in mounts_config["private"].items():
        if "mounts" in config:
            mounts = [
                k for k, v in config["mounts"].items()
                if k in server_vols and v
            ]
            if len(mounts) == 0:
                # Skip project if it has no private mounts
                logging.debug("skipping exports for %s, no private mounts",
                              name)
                continue
        else:
            continue
        ips = get_instance_ips(name, observer_pass, regions, auth_url)
        if ips:
            vol = "misc"
            if name == "tools":
                vol = "tools"
            project = Project(name, config["gid"], ips, vol)
            projects.append(project)
            logging.debug("project %s has %s instances", name,
                          len(project.instance_ips))
        else:
            logging.warning("project %s has no instances; skipping.", name)

    # Validate that there are no duplicate gids
    gids = [p.gid for p in projects]
    if len(set(gids)) != len(gids):
        logging.error("duplicate GIDs found in project config, aborting")
        sys.exit(1)

    logging.warning("found %s projects requiring private mounts",
                    len(projects))
    return projects
Ejemplo n.º 4
0
def get_projects_with_nfs(mounts_config, observer_pass):
    """
    Get populated project objects that need NFS exports
    :param mounts_config: dict
    :returns: list
    """
    projects = []

    # Special one-off session just to grab the list of regions
    session = KeystoneSession(auth=KeystonePassword(
        auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
        username="******",
        password=observer_pass,
        project_name='observer',
        user_domain_name='default',
        project_domain_name='default'))
    keystoneclient = keystone_client.Client(session=session,
                                            interface='public')
    region_recs = keystoneclient.regions.list()
    regions = [region.id for region in region_recs]

    server_vols = mounts_config['volumes_served']
    for name, config in mounts_config['private'].items():
        if 'mounts' in config:
            mounts = [
                k for k, v in config['mounts'].items()
                if k in server_vols and v
            ]
            if len(mounts) == 0:
                # Skip project if it has no private mounts
                logging.debug('skipping exports for %s, no private mounts',
                              name)
                continue
        else:
            continue
        ips = get_instance_ips(name, observer_pass, regions)
        if ips:
            project = Project(name, config['gid'], ips, mounts)
            projects.append(project)
            logging.debug('project %s has %s instances', name,
                          len(project.instance_ips))
        else:
            logging.warning('project %s has no instances; skipping.', name)

    # Validate that there are no duplicate gids
    gids = [p.gid for p in projects]
    if len(set(gids)) != len(gids):
        logging.error('duplicate GIDs found in project config, aborting')
        sys.exit(1)

    logging.warning("found %s projects requiring private mounts",
                    len(projects))
    return projects
Ejemplo n.º 5
0
def get_keystone_session(project_name):

    with open('/etc/novaobserver.yaml') as n:
        nova_observer = yaml.safe_load(n)
        observer_pass = nova_observer['OS_PASSWORD']

    return KeystoneSession(auth=KeystonePassword(
        auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
        username="******",
        password=observer_pass,
        project_name=project_name,
        user_domain_name='default',
        project_domain_name='default'))
Ejemplo n.º 6
0
def get_regions(observer_pass):
    client = keystone_client.Client(
        session=KeystoneSession(auth=KeystonePassword(
            auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
            username="******",
            password=observer_pass,
            project_name="observer",
            user_domain_name="default",
            project_domain_name="default",
        )),
        interface="public",
    )

    return [region.id for region in client.regions.list()]
Ejemplo n.º 7
0
def get_instance_ips(project, observer_pass, regions):
    """
    Return a list of Instance internal IPs for a given project

    This uses the Nova API to fetch this data
    """
    session = KeystoneSession(auth=KeystonePassword(
        auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
        username="******",
        password=observer_pass,
        project_name=project,
        user_domain_name='default',
        project_domain_name='default'))

    ips = []
    for region in regions:
        try:
            client = novaclient.Client(
                "2.0",
                session=session,
                region_name=region,
            )
            for instance in client.servers.list():
                # Only provide internal IPs!
                if 'public' in instance.addresses:
                    # This is a nova-network instance
                    for ip in instance.addresses['public']:
                        if ip['OS-EXT-IPS:type'] == 'fixed' and is_valid_ipv4(
                                ip['addr']):
                            ips.append(str(ip['addr']))
                else:
                    # This is probably a neutron instance.  Export all fixed
                    #  addresses (probably there's only one)
                    for value in instance.addresses.values():
                        for ip in value:
                            if ip['OS-EXT-IPS:type'] == 'fixed' and is_valid_ipv4(
                                    ip['addr']):
                                ips.append(str(ip['addr']))

        except keystoneauth1.exceptions.http.Unauthorized:
            logging.error("Failed to get server list for project %s."
                          "  Maybe the project was deleted." % project)
            ips = []

    return ips
Ejemplo n.º 8
0
def get_instance_ips(project, observer_pass, regions, auth_url):
    """
    Return a list of Instance internal IPs for a given project

    This uses the Nova API to fetch this data
    """
    session = KeystoneSession(auth=KeystonePassword(
        auth_url=auth_url,
        username="******",
        password=observer_pass,
        project_name=project,
        user_domain_name="default",
        project_domain_name="default",
    ))

    ips = []
    for region in regions:
        try:
            client = novaclient.Client("2.0",
                                       session=session,
                                       region_name=region)
            for instance in client.servers.list():
                # Only provide internal IPs!
                if "public" in instance.addresses:
                    # This is a nova-network instance
                    for ip in instance.addresses["public"]:
                        if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
                                ip["addr"]):
                            ips.append(str(ip["addr"]))
                else:
                    # This is probably a neutron instance.  Export all fixed
                    #  addresses (probably there's only one)
                    for value in instance.addresses.values():
                        for ip in value:
                            if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
                                    ip["addr"]):
                                ips.append(str(ip["addr"]))

        except keystoneauth1.exceptions.http.Unauthorized:
            logging.error("Failed to get server list for project %s."
                          "  Maybe the project was deleted." % project)
            raise

    return ips
Ejemplo n.º 9
0
    def drvCreateCloudClient(self, credentials):
        cloudConfig = self.getTargetConfiguration()
        server = cloudConfig['name']
        port = cloudConfig['nova_port']
        projectName = cloudConfig['project_name']
        try:
            session = KeystoneSession()

            def authenticate(self, **kwargs):
                secure = kwargs.pop('secure')
                authUrl = self._authUrl(server, port, secure=secure)
                keystoneCli = KeystoneClient(self.KEYSTONE_API_VERSION,
                                             tenant_name=projectName,
                                             auth_url=authUrl,
                                             username=credentials['username'],
                                             password=credentials['password'],
                                             session=session)
                auth = v2_auth.Password(keystoneCli.auth_url,
                                        username=credentials['username'],
                                        password=credentials['password'])
                session.auth = auth
                keystoneCli.authenticate()
                auth.auth_ref = keystoneCli.auth_ref

                return keystoneCli

            keystoneCli = self._secureToInsecureFallback(authenticate)
            novaCli = self.NovaClientClass(auth_token=keystoneCli.auth_token,
                                           project_id=projectName,
                                           auth_url=keystoneCli.auth_url,
                                           session=session)
            endpoint = session.get_endpoint(service_type="image")
            glanceCli = GlanceClient(self.GLANCE_CLIENT_VERSION,
                                     endpoint=endpoint,
                                     project_id=projectName,
                                     token=keystoneCli.auth_token,
                                     session=session)
            clients = ConsolidatedClient(keystoneCli, novaCli, glanceCli)
        except Exception, e:
            raise errors.PermissionDenied(
                message="Error initializing client: %s" % (e, ))
Ejemplo n.º 10
0
def email_admins(subject, msg):
    keystone_session = KeystoneSession(auth=KeystonePassword(
        auth_url=nova_observer_config['OS_AUTH_URL'],
        username=nova_observer_config['OS_USERNAME'],
        password=nova_observer_config['OS_PASSWORD'],
        project_name=nova_observer_config['OS_PROJECT_NAME'],
        user_domain_name='default',
        project_domain_name='default'))
    keystoneclient = keystone_client.Client(session=keystone_session,
                                            interface='public')
    roleid = None
    for r in keystoneclient.roles.list():
        if r.name == 'projectadmin':
            roleid = r.id
            break

    assert roleid is not None
    for ra in keystoneclient.role_assignments.list(project=project_name,
                                                   role=roleid):
        dn = 'uid={},ou=people,{}'.format(ra.user['id'], ldap_config['basedn'])
        _email_member(dn, subject, msg)
Ejemplo n.º 11
0
#!/usr/bin/python3
from keystoneclient.session import Session as KeystoneSession
from keystoneclient.auth.identity.v3 import Password as KeystonePassword
from novaclient import client as novaclient

client = novaclient.Client(
    "2.0",
    session=KeystoneSession(auth=KeystonePassword(
        auth_url="http://{host}:{port}/v3".format(
            host="cloudcontrol1003.wikimedia.org", port=5000),
        username="******",
        password=open('novaobserver_password').read(
        ),  # public password for guest 'novaobserver' account
        project_name='deployment-prep',
        user_domain_name='default',
        project_domain_name='default')))

for instance in sorted(client.servers.list(),
                       key=lambda instance: instance.name):
    print(instance.name)
Ejemplo n.º 12
0
    help='Exit with 0 if there are no changes and 1 if there are changes. Do not write to file',
    action='store_true'
)

LUA_LINE_TEMPLATE = 'aliasmapping["{public}"] = "{private}" -- {name}\n'

args = argparser.parse_args()
config = yaml.safe_load(args.config_file)

auth = KeystonePassword(
    auth_url=config['nova_api_url'],
    username=config['username'],
    password=config['password'],
    tenant_name=config['admin_project_name']
)
keystoneClient = KeystoneClient(session=KeystoneSession(auth=auth), endpoint=config['nova_api_url'])

projects = []
for tenant in keystoneClient.tenants.list():
    projects.append(tenant.name)

aliases = {}
for project in projects:
    client = novaclient.Client(
        "1.1",
        config['username'],
        config['password'],
        project,
        config['nova_api_url']
    )
Ejemplo n.º 13
0
argparser.add_argument(
    '--check-changes-only',
    help=
    'Exit with 0 if there are no changes and 1 if there are changes. Do not write to file',
    action='store_true')

LUA_LINE_TEMPLATE = '{table}["{key}"] = "{value}" -- {comment}\n'

args = argparser.parse_args()
config = yaml.safe_load(args.config_file)

auth = KeystonePassword(auth_url=config['nova_api_url'],
                        username=config['username'],
                        password=config['password'],
                        tenant_name=config['admin_project_name'])
keystoneClient = KeystoneClient(session=KeystoneSession(auth=auth),
                                endpoint=config['nova_api_url'])

projects = []
for tenant in keystoneClient.tenants.list():
    projects.append(tenant.name)

aliases = {}
for project in projects:
    client = novaclient.Client("1.1", config['username'], config['password'],
                               project, config['nova_api_url'])

    for server in client.servers.list():
        serverAddresses = {}
        try:
            private = [