def upload_uniq_image(osp_connection: Connection, cloud: str, cluster_name: str, images_dir: str, installer: str): """Function uploads unique image to the cluster, instead of making shared one""" inst_url, version = get_url(installer) image_name = f"osia-{cluster_name}-{version}" image_path = Path(images_dir).joinpath(f"rhcos-{version}.qcow2") image_file = None if image_path.exists(): logging.info("Found image at %s", image_path.name) image_file = image_path.as_posix() else: logging.info("Starting download of image %s", inst_url) image_file = download_image(inst_url, image_path.as_posix()) logging.info("Starting upload of image into openstack") osp_connection.create_image(image_name, filename=image_file, container_format="bare", disk_format="qcow2", wait=True, osia_clusters=cluster_name, visibility='private') logging.info("Upload finished") image = osp_connection.image.find_image(image_name) logging.info("Image uploaded as %s", image.name) with open(Path(cluster_name).joinpath("fips.json"), "w") as fips: obj = {'cloud': cloud, 'fips': [], 'image': image_name} json.dump(obj, fips) return image.name
def update_aggregates_db(os_conn=None): if not os_conn: os_conn = OpenStack(session=os.session, cloud=CLOUD, region_name=os.get_region()) aggregates_os = os_conn.list_aggregates() aggregates = {} for aggregate in aggregates_os: for host in aggregate.hosts: aggregates[host] = aggregate.name redis.set('aggregates', json.dumps(aggregates)) redis.set('aggregates:timestamp', time.time())
def _find_cluster_ports(osp_connection: Connection, cluster_name: str) -> Munch: port_list = [k for k in osp_connection.list_ports() if k.name.startswith(cluster_name) and k.name.endswith('ingress-port')] port = next(iter(port_list), None) if port is None: raise Exception(f"Ingress port for cluster {cluster_name} was not found") return port
def get_openstack_connection(region_name): """ Get the OpenStack Connection object. This is the new, all-powerful Python API for OpenStack. It should be used instead of the service-specific APIs such as the Nova API below. The returned Connection object has an attribute for each available service, e.g. "compute", "network", etc. """ profile = Profile() profile.set_region(Profile.ALL, region_name) connection = Connection( profile=profile, user_agent='opencraft-im', auth_url=settings.OPENSTACK_AUTH_URL, project_name=settings.OPENSTACK_TENANT, username=settings.OPENSTACK_USER, password=settings.OPENSTACK_PASSWORD, ) # API queries via the nova client occasionally get connection errors from the OpenStack provider. # To gracefully recover when the unavailability is short-lived, ensure safe requests (as per # urllib3's definition) are retried before giving up. adapter = requests.adapters.HTTPAdapter(max_retries=get_requests_retry()) connection.session.session.mount('http://', adapter) connection.session.session.mount('https://', adapter) return connection
def read_input_inventory(self): auth_url = os.environ.get('OS_AUTH_URL') os_username = os.environ.get('OS_USERNAME') os_project_name = os.environ.get('OS_PROJECT_NAME', os.environ.get('OS_TENANT_NAME')) os_password = os.environ.get('OS_PASSWORD') os_auth_token = os.environ.get('OS_AUTH_TOKEN') os_cacert = os.environ.get('OS_CACERT') ansible_ssh_user = os.environ.get('ANSIBLE_SSH_USER', 'heat-admin') self.osc_conn = Connection() self.undercloud_stack = next(self.osc_conn.orchestration.stacks()) self.plan_name = (os.environ.get('TRIPLEO_PLAN_NAME') or os.environ.get('STACK_NAME_NAME') or self.get_tripleo_plan_name()) session = get_auth_session(auth_url, os_username, os_project_name, os_password, os_auth_token, os_cacert) heat_api_version = ( self.osc_conn.orchestration.get_api_major_version()[0]) self.hclient = heat_client.Client(heat_api_version, session=session) inventory = TripleoInventory(session=session, hclient=self.hclient, auth_url=auth_url, cacert=os_cacert, project_name=os_project_name, username=os_username, ansible_ssh_user=ansible_ssh_user, plan_name=self.plan_name) return inventory.list()
def _find_fit_network(osp_connection: Connection, networks: List[str]) -> Optional[str]: named_networks = {k['name']: k for k in osp_connection.list_networks() if k['name'] in networks} results = dict() for net_name in networks: net_avail = osp_connection.network.get_network_ip_availability(named_networks[net_name]) results[net_name] = net_avail['total_ips'] / net_avail['used_ips'] result = _find_best_fit(results) return (named_networks[result]['id'], result)
def create_client(): return Client(Connection( auth_url=config.openstack_auth_url, domain_name=config.openstack_domain_name, username=config.openstack_admin_username, password=config.openstack_admin_password, project_name=config.openstack_project_name, ))
def make_client(instance): """Returns a instance_ha proxy""" LOG.debug('Instantiating masakari service client') con = Connection(session=instance.session, interface=instance.interface, region_name=instance.region_name, ha_api_version=instance._api_version[API_NAME]) return con.instance_ha
def resolve_image(osp_connection: Connection, cloud: str, cluster_name: str, images_dir: str, installer: str, error: Optional[Exception]): """Function searches for image in openstack and creates it if it doesn't exist""" inst_url, version = get_url(installer) image_name = f"osia-rhcos-{version}" image = osp_connection.image.find_image(image_name, ignore_missing=True) if image is None: image_path = Path(images_dir).joinpath(f"rhcos-{version}.qcow2") image_file = None if image_path.exists(): logging.info("Found image at %s", image_path.name) image_file = image_path.as_posix() else: logging.info("Starting download of image %s", inst_url) image_file = download_image(inst_url, image_path.as_posix()) logging.info("Starting upload of image into openstack") osp_connection.create_image(image_name, filename=image_file, container_format="bare", disk_format="qcow2", wait=True, osia_clusters=cluster_name, visibility='private') logging.info("Upload finished") image = osp_connection.image.find_image(image_name) logging.info("Image uploaded as %s", image.name) else: logging.info("Reusing found image in openstack %s", image.name) try: add_cluster(osp_connection, image, cluster_name) except SDKException as err: if error is not None: raise ImageException("Couldn't add cluster to image") from err logging.warning( "Image disappeared while metadata were written, trying again") logging.debug("Openstack error: %s", err) return resolve_image(osp_connection, cloud, cluster_name, images_dir, installer, err) with open(Path(cluster_name).joinpath("fips.json"), "w") as fips: obj = {'cloud': cloud, 'fips': [], 'image': image_name} json.dump(obj, fips) return image.name
def os_connect(): prof = Profile() prof.set_region(Profile.ALL, settings.OS_REGION_NAME) return Connection(profile=prof, user_agent='cloudvcl', auth_url=settings.OS_AUTH_URL, project_name=settings.OS_PROJECT_NAME, username=settings.OS_USERNAME, password=settings.OS_PASSWORD, user_domain_name='default', project_domain_name='default')
def __init__(self): self.ansible = AnsibleModule( swift_full_argument_spec(**self.argument_spec), **self.module_kwargs) self.params = self.ansible.params self.module_name = self.ansible._name self.results = {'changed': False} self.exit = self.exit_json = self.ansible.exit_json self.fail = self.fail_json = self.ansible.fail_json self.cloud = OpenStackConfig().get_one() self.client = Connection(config=self.cloud).object_store
def generate_env_specific_variables(self): endpointmap_resource = self.hclient.resources.get( self.plan_name, 'EndpointMap') endpointmap_resource_dict = endpointmap_resource.to_dict( )['attributes']['endpoint_map'] self.internal_lb_vip = endpointmap_resource_dict['KeystoneInternal'][ 'host'] self.external_lb_vip = endpointmap_resource_dict['KeystonePublic'][ 'host'] stack_env_dict = self.osc_conn.orchestration.get_stack_environment( self.plan_name).to_dict()['parameter_defaults'] # get galera root password self.galera_password = stack_env_dict['MysqlRootPassword'] # load overcloud env and osc self.load_rc_file(stack_name=self.plan_name) tmp_osc_conn = Connection() # get cinder_backend_volume fact self.cinder_backend_fact = {} for cinder_backend_pool in tmp_osc_conn.volume.backend_pools(): backend_host = cinder_backend_pool.name.split('#')[0] # (NOTE:tonytan4ever): skip transient legacy backends if 'legacy' in backend_host: continue cinder_backend_pool_dict = cinder_backend_pool.to_dict( )['capabilities'] if 'solidfire' in cinder_backend_pool_dict['volume_backend_name']: self.cinder_backend_fact['solidfire'] = { 'volume_driver': 'abc', 'host': backend_host } if 'netapp' in cinder_backend_pool_dict['volume_backend_name']: self.cinder_backend_fact['netapp'] = { 'volume_driver': 'abc', 'host': backend_host } if 'ceph' in cinder_backend_pool_dict['volume_backend_name']: self.cinder_backend_fact['ceph'] = { 'volume_driver': 'abc', 'host': backend_host } # reset to undercloud rc for osc self.load_rc_file()
def _find_fit_network( osp_connection: Connection, networks: List[str]) -> Tuple[Optional[str], Optional[str]]: named_networks = { k['name']: k for k in osp_connection.list_networks() if k['name'] in networks } results = {} for net_name in networks: net_avail = osp_connection.network.get_network_ip_availability( named_networks[net_name]) subnet_usage = [(subnet['total_ips'], subnet['used_ips']) for subnet in net_avail.subnet_ip_availability if subnet['ip_version'] == 4] total_ips, used_ips = [sum(i) for i in zip(*subnet_usage)] results[net_name] = total_ips / used_ips result = _find_best_fit(results) return named_networks[result]['id'], result
cloud = 'default' # (NOTE:tonytan4ever): for OSP cloud, cloud parameter needs to be None if os.getenv('OS_CLOUDNAME') is not None: cloud = None if os.path.exists(OPENRC) or os.path.exists(STACKRC): try: OSC_CONFIG = get_cloud_region(cloud=cloud) except ConfigException as e: # (NOTE:tonytan4ever) Liberty cloud does not have 'default' config if 'Cloud default was not found' in str(e): OSC_CONFIG = get_cloud_region(cloud=None) else: raise e OSC_CLIENT = Connection(config=OSC_CONFIG, verify=False) def get_os_component_major_api_version(component_name): supported_os_component_osc_mapping = { 'nova': 'compute', 'neutron': 'network', 'cinder': 'volume', 'glance': 'image', 'keystone': 'identity', 'ironic': 'baremetal_introspection', 'heat': 'orchestration', 'swfit': 'object_store' } if component_name not in supported_os_component_osc_mapping.keys(): raise NameError("Not a supported OpenStack component name: %s,"
def resurrect_instances(dead_hv, spare_hv, update_db=True): assert dead_hv != spare_hv os_conn = OpenStack(session=os.session, cloud=CLOUD, region_name=os.get_region()) if update_db: _logger.info('updating redis database') refresh_redis_inventory(True) spare_hv_r = redis.get('hypervisors', json.loads)[spare_hv] assert spare_hv_r['running_vms'] == 0 dead_service = spare_service = None for svc in os_conn.compute.services(): if svc.host == dead_hv: dead_service = svc elif svc.host == spare_hv: spare_service = svc assert svc.status == 'disabled' assert dead_service is not None assert spare_service is not None assert spare_service.state == 'up' assert spare_service.zone == dead_service.zone assert 'spare' in spare_service.disables_reason.lower() _logger.info(f'verifying that {dead_hv} is dead') if not nmap_scan([dead_hv], [22, 111, 16509]): raise Exception(f'hypervisor {dead_hv} does not seem to be dead!') instance_list = [] servers = json.loads(redis.get('servers')) for _, server in servers.items(): if server['hypervisor_hostname'] == spare_hv: raise Exception(f'spare hypervisor {spare_hv} has vms assigned!') if server['hypervisor_hostname'] == dead_hv: instance_list.append(server['id']) server['hypervisor_hostname'] = spare_hv if not instance_list: _logger.warning(f'{dead_hv} does not run any instances') return _logger.info('updating database records in nova') db_conn = mysql_connect(host=MYSQL_HOST, user=MYSQL_USER, passwd=MYSQL_PASS, db='nova') try: spare_hv = escape_string(spare_hv) with db_conn.cursor() as cursor: for uuid in instance_list: uuid = escape_string(uuid) query = \ f'''update instances set host = "{spare_hv}", node = "{spare_hv}" where uuid="{uuid}"''' _logger.debug(query) cursor.execute(query) db_conn.commit() finally: db_conn.close() _logger.info('updating servers inventory db') redis.set('servers', json.dumps(servers)) exceptions = [] for uuid in instance_list: try: if servers[uuid]['vm_state'] == 'stopped': _logger.info(f'instance {uuid} is stoppped, not rebooting') continue _logger.info(f'hard rebooting instance {uuid}') os_conn.compute.reboot_server(uuid, 'HARD') for ifce in os_conn.compute.server_interfaces(uuid): _logger.info(f'updating port binding on {ifce.port_id}') port = os_conn.get_port(ifce.port_id) if port: os_conn.network.update_port( port, **{'binding:host_id': spare_hv}) except Exception as e: exceptions.append(str(e)) if exceptions: raise Exception('\n'.join(exceptions)) _logger.info(f'disabling nova on {dead_hv}, enabling nova on {spare_hv}') os_conn.compute.disable_service(dead_service, dead_hv, 'nova-compute', f'sonny resurrection on {spare_hv}') os_conn.compute.enable_service(spare_service, spare_hv, 'nova-compute')