def list(self, backend_type=None): """ Overview of all backends (from a certain type, if given) on the local node (or a remote one) """ if backend_type is None: return BackendList.get_backends() return BackendTypeList.get_backend_type_by_code(backend_type).backends
def list(self, query=None): """ Overview of all backend types """ if query is not None: query = json.loads(query) return DataList(BackendType, query) return BackendTypeList.get_backend_types()
def list(self, query=None): """ Overview of all backend types :param query: Optional filter for BackendTypes :type query: DataQuery """ if query is not None: return DataList(BackendType, query) return BackendTypeList.get_backend_types()
def get_backendtype_by_code(code): """ Retrieve the Backend Type based on its code :param code: Code of the Backend Type :return: Backend Type DAL object """ if code not in GeneralBackend.get_valid_backendtypes(): raise ValueError('Unsupported backend type {0} provided'.format(code)) return BackendTypeList.get_backend_type_by_code(code=code)
def get(self, request, *args, **kwargs): """ Fetches metadata """ _ = args, kwargs data = {'authenticated': False, 'authentication_state': None, 'username': None, 'userguid': None, 'roles': [], 'storagerouter_ips': [sr.ip for sr in StorageRouterList.get_storagerouters()], 'versions': list(settings.VERSION), 'plugins': {}} try: # Gather plugin metadata plugins = {} # - Backends. BackendType plugins must set the has_plugin flag on True for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] data['plugins'] = plugins # Gather authorization metadata if 'HTTP_AUTHORIZATION' not in request.META: return HttpResponse, dict(data.items() + {'authentication_state': 'unauthenticated'}.items()) authorization_type, access_token = request.META['HTTP_AUTHORIZATION'].split(' ') if authorization_type != 'Bearer': return HttpResponse, dict(data.items() + {'authentication_state': 'invalid_authorization_type'}.items()) tokens = BearerTokenList.get_by_access_token(access_token) if len(tokens) != 1: return HttpResponse, dict(data.items() + {'authentication_state': 'invalid_token'}.items()) token = tokens[0] if token.expiration < time.time(): for junction in token.roles.itersafe(): junction.delete() token.delete() return HttpResponse, dict(data.items() + {'authentication_state': 'token_expired'}.items()) # Gather user metadata user = token.client.user if not user.is_active: return HttpResponse, dict(data.items() + {'authentication_state': 'inactive_user'}.items()) roles = [j.role.code for j in token.roles] return HttpResponse, dict(data.items() + {'authenticated': True, 'authentication_state': 'authenticated', 'username': user.username, 'userguid': user.guid, 'roles': roles, 'plugins': plugins}.items()) except Exception as ex: logger.exception('Unexpected exception: {0}'.format(ex)) return HttpResponse, dict(data.items() + {'authentication_state': 'unexpected_exception'}.items())
def list(self, query=None): """ Overview of all backend types """ if query is not None: query = json.loads(query) query_result = DataList({'object': BackendType, 'data': DataList.select.GUIDS, 'query': query}).data return DataObjectList(query_result, BackendType) return BackendTypeList.get_backend_types()
def get_backendtype_guid_by_code(backendtype_code): """ Get a backend type guid by a backend code :param backendtype_code: type name of a backend :type backendtype_code: str :return: backendtype_guid :rtype: str """ return BackendTypeList.get_backend_type_by_code(backendtype_code).guid
def list(self, request, backend_type=None): """ Overview of all backends (from a certain type, if given) on the local node (or a remote one) :param request: The raw request :type request: Request :param backend_type: Optional BackendType code to filter :type backend_type: str """ if backend_type is None: possible_backends = BackendList.get_backends() else: possible_backends = BackendTypeList.get_backend_type_by_code(backend_type).backends backends = [] for backend in possible_backends: if Toolbox.access_granted(request.client, user_rights=backend.user_rights, client_rights=backend.client_rights): backends.append(backend) return backends
def list(self, backend_type=None, ip=None, port=None, client_id=None, client_secret=None, contents=None): """ Overview of all backends (from a certain type, if given) on the local node (or a remote one) """ if ip is None: if backend_type is None: return BackendList.get_backends() return BackendTypeList.get_backend_type_by_code(backend_type).backends client = OVSClient(ip, port, client_id, client_secret) try: remote_backends = client.get( "/backends/", params={"backend_type": backend_type, "contents": "" if contents is None else contents} ) except (HTTPError, URLError): raise NotAcceptable("Could not load remote backends") backend_list = [] for entry in remote_backends["data"]: backend = type("Backend", (), entry)() backend_list.append(backend) return backend_list
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at version 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 0.0.1 introduced: if working_version < 1: # Add backends for backend_type_info in [('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in [ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # We're now at version 0.0.1 working_version = 1 # Version 0.0.2 introduced: if working_version < 2: # Execute some code that upgrades to version 2 # working_version = 2 pass return working_version
def add_vpool(parameters): """ Add a vPool to the machine this task is running on """ parameters = {} if parameters is None else parameters ip = parameters['storagerouter_ip'] vpool_name = parameters['vpool_name'] if StorageRouterController._validate_ip(ip) is False: raise ValueError('The entered ip address is invalid') if not re.match('^[0-9a-z]+(\-+[0-9a-z]+)*$', vpool_name): raise ValueError('Invalid vpool_name given') client = SSHClient.load(ip) # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish unique_id = System.get_my_machine_id(client) storagerouter = None for current_storagerouter in StorageRouterList.get_storagerouters(): if current_storagerouter.ip == ip and current_storagerouter.machine_id == unique_id: storagerouter = current_storagerouter break if storagerouter is None: raise RuntimeError('Could not find Storage Router with given ip address') vpool = VPoolList.get_vpool_by_name(vpool_name) storagedriver = None if vpool is not None: if vpool.backend_type.code == 'local': # Might be an issue, investigating whether it's on the same not or not if len(vpool.storagedrivers) == 1 and vpool.storagedrivers[0].storagerouter.machine_id != unique_id: raise RuntimeError('A local vPool with name {0} already exists'.format(vpool_name)) for vpool_storagedriver in vpool.storagedrivers: if vpool_storagedriver.storagerouter_guid == storagerouter.guid: storagedriver = vpool_storagedriver # The vPool is already added to this Storage Router and this might be a cleanup/recovery # Check whether there are running machines on this vPool machine_guids = [] for vdisk in vpool.vdisks: if vdisk.vmachine_guid not in machine_guids: machine_guids.append(vdisk.vmachine_guid) if vdisk.vmachine.hypervisor_status in ['RUNNING', 'PAUSED']: raise RuntimeError( 'At least one vMachine using this vPool is still running or paused. Make sure there are no active vMachines' ) nodes = {ip} # Set comprehension if vpool is not None: for vpool_storagedriver in vpool.storagedrivers: nodes.add(vpool_storagedriver.storagerouter.ip) nodes = list(nodes) services = ['volumedriver_{0}'.format(vpool_name), 'failovercache_{0}'.format(vpool_name)] # Stop services for node in nodes: node_client = SSHClient.load(node) for service in services: System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service if Service.has_service('{0}'): Service.disable_service('{0}') """.format(service)) System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service if Service.has_service('{0}'): Service.stop_service('{0}') """.format(service)) # Keep in mind that if the Storage Driver exists, the vPool does as well client = SSHClient.load(ip) mountpoint_bfs = '' directories_to_create = [] if vpool is None: vpool = VPool() supported_backends = System.read_remote_config(client, 'volumedriver.supported.backends').split(',') if 'rest' in supported_backends: supported_backends.remove('rest') # REST is not supported for now backend_type = BackendTypeList.get_backend_type_by_code(parameters['type']) vpool.backend_type = backend_type connection_host = connection_port = connection_username = connection_password = None if vpool.backend_type.code in ['local', 'distributed']: vpool.metadata = {'backend_type': 'LOCAL'} mountpoint_bfs = parameters['mountpoint_bfs'] directories_to_create.append(mountpoint_bfs) vpool.metadata['local_connection_path'] = mountpoint_bfs if vpool.backend_type.code == 'rest': connection_host = parameters['connection_host'] connection_port = parameters['connection_port'] rest_connection_timeout_secs = parameters['connection_timeout'] vpool.metadata = {'rest_connection_host': connection_host, 'rest_connection_port': connection_port, 'buchla_connection_log_level': "0", 'rest_connection_verbose_logging': rest_connection_timeout_secs, 'rest_connection_metadata_format': "JSON", 'backend_type': 'REST'} elif vpool.backend_type.code in ('ceph_s3', 'amazon_s3', 'swift_s3'): connection_host = parameters['connection_host'] connection_port = parameters['connection_port'] connection_username = parameters['connection_username'] connection_password = parameters['connection_password'] if vpool.backend_type.code in ['swift_s3']: strict_consistency = 'false' s3_connection_flavour = 'SWIFT' else: strict_consistency = 'true' s3_connection_flavour = 'S3' vpool.metadata = {'s3_connection_host': connection_host, 's3_connection_port': connection_port, 's3_connection_username': connection_username, 's3_connection_password': connection_password, 's3_connection_flavour': s3_connection_flavour, 's3_connection_strict_consistency': strict_consistency, 's3_connection_verbose_logging': 1, 'backend_type': 'S3'} vpool.name = vpool_name vpool.description = "{} {}".format(vpool.backend_type.code, vpool_name) vpool.login = connection_username vpool.password = connection_password if not connection_host: vpool.connection = None else: vpool.connection = '{}:{}'.format(connection_host, connection_port) vpool.save() # Connection information is Storage Driver related information new_storagedriver = False if storagedriver is None: storagedriver = StorageDriver() new_storagedriver = True mountpoint_temp = parameters['mountpoint_temp'] mountpoint_md = parameters['mountpoint_md'] mountpoint_readcache1 = parameters['mountpoint_readcache1'] mountpoint_readcache2 = parameters.get('mountpoint_readcache2', '') mountpoint_writecache = parameters['mountpoint_writecache'] mountpoint_foc = parameters['mountpoint_foc'] directories_to_create.append(mountpoint_temp) directories_to_create.append(mountpoint_md) directories_to_create.append(mountpoint_readcache1) if mountpoint_readcache2: directories_to_create.append(mountpoint_readcache2) directories_to_create.append(mountpoint_writecache) directories_to_create.append(mountpoint_foc) client = SSHClient.load(ip) dir_create_script = """ import os for directory in {0}: if not os.path.exists(directory): os.makedirs(directory) """.format(directories_to_create) System.exec_remote_python(client, dir_create_script) read_cache1_fs = os.statvfs(mountpoint_readcache1) read_cache2_fs = None if mountpoint_readcache2: read_cache2_fs = os.statvfs(mountpoint_readcache2) write_cache_fs = os.statvfs(mountpoint_writecache) fdcache = '{}/fd_{}'.format(mountpoint_writecache, vpool_name) scocache = '{}/sco_{}'.format(mountpoint_writecache, vpool_name) readcache1 = '{}/read1_{}'.format(mountpoint_readcache1, vpool_name) files2create = [readcache1] if mountpoint_readcache2 and mountpoint_readcache1 != mountpoint_readcache2: readcache2 = '{}/read2_{}'.format(mountpoint_readcache2, vpool_name) files2create.append(readcache2) else: readcache2 = '' failovercache = '{}/foc_{}'.format(mountpoint_foc, vpool_name) metadatapath = '{}/metadata_{}'.format(mountpoint_md, vpool_name) tlogpath = '{}/tlogs_{}'.format(mountpoint_md, vpool_name) rsppath = '/var/rsp/{}'.format(vpool_name) dirs2create = [scocache, failovercache, metadatapath, tlogpath, rsppath, System.read_remote_config(client, 'volumedriver.readcache.serialization.path')] cmd = "cat /etc/mtab | grep ^/dev/ | cut -d ' ' -f 2" mountpoints = [device.strip() for device in client.run(cmd).strip().split('\n')] mountpoints.remove('/') def is_partition(directory): for mountpoint in mountpoints: if directory == mountpoint: return True return False # Cache sizes # 20% = scocache # 20% = failovercache (@TODO: check if this can possibly consume more than 20%) # 60% = readcache # safety values: readcache1_factor = 0.2 readcache2_factor = 0.2 writecache_factor = 0.1 if (mountpoint_readcache1 == mountpoint_readcache2) or not mountpoint_readcache2: delta = set() delta.add(mountpoint_readcache1 if is_partition(mountpoint_readcache1) else '/dummy') delta.add(mountpoint_writecache if is_partition(mountpoint_writecache) else '/dummy') delta.add(mountpoint_foc if is_partition(mountpoint_foc) else '/dummy') if len(delta) == 1: readcache1_factor = 0.49 writecache_factor = 0.2 elif len(delta) == 2: if mountpoint_writecache == mountpoint_foc: readcache1_factor = 0.98 writecache_factor = 0.49 else: readcache1_factor = 0.49 if mountpoint_readcache1 == mountpoint_writecache: writecache_factor = 0.49 else: writecache_factor = 0.98 elif len(delta) == 3: readcache1_factor = 0.98 writecache_factor = 0.98 else: delta = set() delta.add(mountpoint_readcache1 if is_partition(mountpoint_readcache1) else '/dummy') delta.add(mountpoint_readcache2 if is_partition(mountpoint_readcache2) else '/dummy') delta.add(mountpoint_writecache if is_partition(mountpoint_writecache) else '/dummy') delta.add(mountpoint_foc if is_partition(mountpoint_foc) else '/dummy') if len(delta) == 1: # consider them all to be directories readcache1_factor = 0.24 readcache2_factor = 0.24 writecache_factor = 0.24 elif len(delta) == 2: if mountpoint_writecache == mountpoint_foc: writecache_factor = 0.24 if mountpoint_readcache1 == mountpoint_writecache: readcache1_factor = 0.49 readcache2_factor = 0.98 else: readcache1_factor = 0.98 readcache2_factor = 0.49 else: readcache1_factor = readcache2_factor = 0.49 writecache_factor = 0.49 elif len(delta) == 3: if mountpoint_writecache == mountpoint_foc: readcache1_factor = 0.98 readcache2_factor = 0.98 writecache_factor = 0.49 elif mountpoint_readcache1 == mountpoint_writecache: readcache1_factor = 0.49 readcache2_factor = 0.98 writecache_factor = 0.49 elif mountpoint_readcache1 == mountpoint_foc: readcache1_factor = 0.49 readcache2_factor = 0.98 writecache_factor = 0.98 elif mountpoint_readcache2 == mountpoint_writecache: readcache1_factor = 0.98 readcache2_factor = 0.49 writecache_factor = 0.49 elif mountpoint_readcache2 == mountpoint_foc: readcache1_factor = 0.98 readcache2_factor = 0.49 writecache_factor = 0.98 elif len(delta) == 4: readcache1_factor = 0.98 readcache2_factor = 0.98 writecache_factor = 0.98 # summarize caching on root partition (directory only) root_assigned = dict() if not is_partition(mountpoint_readcache1): root_assigned['readcache1_factor'] = readcache1_factor if not is_partition(mountpoint_readcache2): root_assigned['readcache2_factor'] = readcache2_factor if not is_partition(mountpoint_writecache): root_assigned['writecache_factor'] = writecache_factor if not is_partition(mountpoint_foc): root_assigned['foc_factor'] = min(readcache1_factor, readcache2_factor, writecache_factor) # always leave at least 20% of free space division_factor = 1.0 total_size = sum(root_assigned.values()) + .02 * len(root_assigned) if 0.8 < total_size < 1.6: division_factor = 2.0 elif 1.6 < total_size < 3.2: division_factor = 4.0 elif total_size >= 3.2: division_factor = 8.0 if 'readcache1_factor' in root_assigned.keys(): readcache1_factor /= division_factor if 'readcache2_factor' in root_assigned.keys(): readcache2_factor /= division_factor if 'writecache_factor' in root_assigned.keys(): writecache_factor /= division_factor scocache_size = '{0}KiB'.format((int(write_cache_fs.f_bavail * writecache_factor / 4096) * 4096) * 4) if (mountpoint_readcache1 and not mountpoint_readcache2) or (mountpoint_readcache1 == mountpoint_readcache2): mountpoint_readcache2 = '' readcache1_size = '{0}KiB'.format((int(read_cache1_fs.f_bavail * readcache1_factor / 4096) * 4096) * 4) readcache2 = '' readcache2_size = '0KiB' else: readcache1_size = '{0}KiB'.format((int(read_cache1_fs.f_bavail * readcache1_factor / 4096) * 4096) * 4) readcache2_size = '{0}KiB'.format((int(read_cache2_fs.f_bavail * readcache2_factor / 4096) * 4096) * 4) if new_storagedriver: ports_in_use = System.ports_in_use(client) ports_reserved = [] ports_in_use_model = {} for port_storagedriver in StorageDriverList.get_storagedrivers(): if port_storagedriver.vpool_guid not in ports_in_use_model: ports_in_use_model[port_storagedriver.vpool_guid] = port_storagedriver.ports ports_reserved += port_storagedriver.ports if vpool.guid in ports_in_use_model: # The vPool is extended to another StorageRouter. We need to use these ports. ports = ports_in_use_model[vpool.guid] if any(port in ports_in_use for port in ports): raise RuntimeError('The required ports are in use') else: # First StorageDriver for this vPool, so generating new ports ports = [] for port_range in System.read_remote_config(client, 'volumedriver.filesystem.ports').split(','): port_range = port_range.strip() if '-' in port_range: current_range = (int(port_range.split('-')[0]), int(port_range.split('-')[1])) else: current_range = (int(port_range), 65536) current_port = current_range[0] while len(ports) < 3: if current_port not in ports_in_use and current_port not in ports_reserved: ports.append(current_port) current_port += 1 if current_port > current_range[1]: break if len(ports) != 3: raise RuntimeError('Could not find enough free ports') else: ports = storagedriver.ports ip_path = Configuration.get('ovs.core.ip.path') if ip_path is None: ip_path = "`which ip`" cmd = "{0} a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1".format(ip_path) ipaddresses = client.run(cmd).strip().split('\n') ipaddresses = [ipaddr.strip() for ipaddr in ipaddresses] grid_ip = System.read_remote_config(client, 'ovs.grid.ip') if grid_ip in ipaddresses: ipaddresses.remove(grid_ip) if not ipaddresses: raise RuntimeError('No available ip addresses found suitable for Storage Router storage ip') if storagerouter.pmachine.hvtype == 'KVM': volumedriver_storageip = '127.0.0.1' else: volumedriver_storageip = parameters['storage_ip'] vrouter_id = '{0}{1}'.format(vpool_name, unique_id) vrouter_config = {'vrouter_id': vrouter_id, 'vrouter_redirect_timeout_ms': '5000', 'vrouter_routing_retries': 10, 'vrouter_volume_read_threshold': 1024, 'vrouter_volume_write_threshold': 1024, 'vrouter_file_read_threshold': 1024, 'vrouter_file_write_threshold': 1024, 'vrouter_min_workers': 4, 'vrouter_max_workers': 16} voldrv_arakoon_cluster_id = str(System.read_remote_config(client, 'volumedriver.arakoon.clusterid')) voldrv_arakoon_cluster = ArakoonManagementEx().getCluster(voldrv_arakoon_cluster_id) voldrv_arakoon_client_config = voldrv_arakoon_cluster.getClientConfig() arakoon_node_configs = [] for arakoon_node in voldrv_arakoon_client_config.keys(): arakoon_node_configs.append(ArakoonNodeConfig(arakoon_node, voldrv_arakoon_client_config[arakoon_node][0][0], voldrv_arakoon_client_config[arakoon_node][1])) vrouter_clusterregistry = ClusterRegistry(str(vpool.guid), voldrv_arakoon_cluster_id, arakoon_node_configs) node_configs = [] for existing_storagedriver in StorageDriverList.get_storagedrivers(): if existing_storagedriver.vpool_guid == vpool.guid: node_configs.append(ClusterNodeConfig(str(existing_storagedriver.storagedriver_id), str(existing_storagedriver.cluster_ip), existing_storagedriver.ports[0], existing_storagedriver.ports[1], existing_storagedriver.ports[2])) if new_storagedriver: node_configs.append(ClusterNodeConfig(vrouter_id, grid_ip, ports[0], ports[1], ports[2])) vrouter_clusterregistry.set_node_configs(node_configs) readcaches = [{'path': readcache1, 'size': readcache1_size}] if readcache2: readcaches.append({'path': readcache2, 'size': readcache2_size}) scocaches = [{'path': scocache, 'size': scocache_size}] filesystem_config = {'fs_backend_path': mountpoint_bfs} volumemanager_config = {'metadata_path': metadatapath, 'tlog_path': tlogpath} storagedriver_config_script = """ from ovs.plugin.provider.configuration import Configuration from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration fd_config = {{'fd_cache_path': '{11}', 'fd_extent_cache_capacity': '1024', 'fd_namespace' : 'fd-{0}-{12}'}} storagedriver_configuration = StorageDriverConfiguration('{0}') storagedriver_configuration.configure_backend({1}) storagedriver_configuration.configure_readcache({2}, Configuration.get('volumedriver.readcache.serialization.path') + '/{0}') storagedriver_configuration.configure_scocache({3}, '1GB', '2GB') storagedriver_configuration.configure_failovercache('{4}') storagedriver_configuration.configure_filesystem({5}) storagedriver_configuration.configure_volumemanager({6}) storagedriver_configuration.configure_volumerouter('{12}', {7}) storagedriver_configuration.configure_arakoon_cluster('{8}', {9}) storagedriver_configuration.configure_hypervisor('{10}') storagedriver_configuration.configure_filedriver(fd_config) """.format(vpool_name, vpool.metadata, readcaches, scocaches, failovercache, filesystem_config, volumemanager_config, vrouter_config, voldrv_arakoon_cluster_id, voldrv_arakoon_client_config, storagerouter.pmachine.hvtype, fdcache, vpool.guid) System.exec_remote_python(client, storagedriver_config_script) remote_script = """ import os from configobj import ConfigObj from ovs.plugin.provider.configuration import Configuration protocol = Configuration.get('ovs.core.broker.protocol') login = Configuration.get('ovs.core.broker.login') password = Configuration.get('ovs.core.broker.password') vpool_name = {0} uris = [] cfg = ConfigObj('/opt/OpenvStorage/config/rabbitmqclient.cfg') main_section = cfg.get('main') nodes = main_section['nodes'] if type(main_section['nodes']) == list else [main_section['nodes']] for node in nodes: uris.append({{'amqp_uri': '{{0}}://{{1}}:{{2}}@{{3}}'.format(protocol, login, password, cfg.get(node)['location'])}}) from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration queue_config = {{'events_amqp_routing_key': Configuration.get('ovs.core.broker.volumerouter.queue'), 'events_amqp_uris': uris}} for config_file in os.listdir('/opt/OpenvStorage/config/voldrv_vpools'): this_vpool_name = config_file.replace('.json', '') if config_file.endswith('.json') and (vpool_name is None or vpool_name == this_vpool_name): storagedriver_configuration = StorageDriverConfiguration(this_vpool_name) storagedriver_configuration.configure_event_publisher(queue_config) """.format(vpool_name if vpool_name is None else "'{0}'".format(vpool_name)) System.exec_remote_python(client, remote_script) # Updating the model storagedriver.storagedriver_id = vrouter_id storagedriver.name = vrouter_id.replace('_', ' ') storagedriver.description = storagedriver.name storagedriver.storage_ip = volumedriver_storageip storagedriver.cluster_ip = grid_ip storagedriver.ports = ports storagedriver.mountpoint = '/mnt/{0}'.format(vpool_name) storagedriver.mountpoint_temp = mountpoint_temp storagedriver.mountpoint_readcache1 = mountpoint_readcache1 storagedriver.mountpoint_readcache2 = mountpoint_readcache2 storagedriver.mountpoint_writecache = mountpoint_writecache storagedriver.mountpoint_foc = mountpoint_foc storagedriver.mountpoint_bfs = mountpoint_bfs storagedriver.mountpoint_md = mountpoint_md storagedriver.storagerouter = storagerouter storagedriver.vpool = vpool storagedriver.save() dirs2create.append(storagedriver.mountpoint) dirs2create.append(mountpoint_writecache + '/' + '/fd_' + vpool_name) dirs2create.append('{0}/fd_{1}'.format(mountpoint_writecache, vpool_name)) file_create_script = """ import os for directory in {0}: if not os.path.exists(directory): os.makedirs(directory) for filename in {1}: if not os.path.exists(filename): open(filename, 'a').close() """.format(dirs2create, files2create) System.exec_remote_python(client, file_create_script) voldrv_config_file = '{0}/voldrv_vpools/{1}.json'.format(System.read_remote_config(client, 'ovs.core.cfgdir'), vpool_name) log_file = '/var/log/ovs/volumedriver/{0}.log'.format(vpool_name) vd_cmd = '/usr/bin/volumedriver_fs -f --config-file={0} --mountpoint {1} --logrotation --logfile {2} -o big_writes -o sync_read -o allow_other'.format( voldrv_config_file, storagedriver.mountpoint, log_file) if storagerouter.pmachine.hvtype == 'KVM': vd_stopcmd = 'umount {0}'.format(storagedriver.mountpoint) else: vd_stopcmd = 'exportfs -u *:{0}; umount {0}'.format(storagedriver.mountpoint) vd_name = 'volumedriver_{}'.format(vpool_name) log_file = '/var/log/ovs/volumedriver/foc_{0}.log'.format(vpool_name) fc_cmd = '/usr/bin/failovercachehelper --config-file={0} --logfile={1}'.format(voldrv_config_file, log_file) fc_name = 'failovercache_{0}'.format(vpool_name) params = {'<VPOOL_MOUNTPOINT>': storagedriver.mountpoint, '<HYPERVISOR_TYPE>': storagerouter.pmachine.hvtype, '<VPOOL_NAME>': vpool_name, '<UUID>': str(uuid.uuid4())} if Osdist.is_ubuntu(client): if client.file_exists('/opt/OpenvStorage/config/templates/upstart/ovs-volumedriver.conf'): client.run('cp -f /opt/OpenvStorage/config/templates/upstart/ovs-volumedriver.conf /opt/OpenvStorage/config/templates/upstart/ovs-volumedriver_{0}.conf'.format(vpool_name)) client.run('cp -f /opt/OpenvStorage/config/templates/upstart/ovs-failovercache.conf /opt/OpenvStorage/config/templates/upstart/ovs-failovercache_{0}.conf'.format(vpool_name)) else: if client.file_exists('/opt/OpenvStorage/config/templates/systemd/ovs-volumedriver.service'): client.run('cp -f /opt/OpenvStorage/config/templates/systemd/ovs-volumedriver.service /opt/OpenvStorage/config/templates/systemd/ovs-volumedriver_{0}.service'.format(vpool_name)) client.run('cp -f /opt/OpenvStorage/config/templates/systemd/ovs-failovercache.service /opt/OpenvStorage/config/templates/systemd/ovs-failovercache_{0}.service'.format(vpool_name)) service_script = """ from ovs.plugin.provider.service import Service Service.add_service(package=('openvstorage', 'volumedriver'), name='{0}', command='{1}', stop_command='{2}', params={5}) Service.add_service(package=('openvstorage', 'failovercache'), name='{3}', command='{4}', stop_command=None, params={5}) """.format( vd_name, vd_cmd, vd_stopcmd, fc_name, fc_cmd, params ) System.exec_remote_python(client, service_script) if storagerouter.pmachine.hvtype == 'VMWARE': client.run("grep -q '/tmp localhost(ro,no_subtree_check)' /etc/exports || echo '/tmp localhost(ro,no_subtree_check)' >> /etc/exports") if Osdist.is_ubuntu(client): client.run('service nfs-kernel-server start') else: client.run('service nfs start') if storagerouter.pmachine.hvtype == 'KVM': client.run('virsh pool-define-as {0} dir - - - - {1}'.format(vpool_name, storagedriver.mountpoint)) client.run('virsh pool-build {0}'.format(vpool_name)) client.run('virsh pool-start {0}'.format(vpool_name)) client.run('virsh pool-autostart {0}'.format(vpool_name)) # Start services for node in nodes: node_client = SSHClient.load(node) for service in services: System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service Service.enable_service('{0}') """.format(service)) System.exec_remote_python(node_client, """ from ovs.plugin.provider.service import Service Service.start_service('{0}') """.format(service)) # Fill vPool size vfs_info = os.statvfs('/mnt/{0}'.format(vpool_name)) vpool.size = vfs_info.f_blocks * vfs_info.f_bsize vpool.save() # Configure Cinder ovsdb = PersistentFactory.get_client() vpool_config_key = str('ovs_openstack_cinder_%s' % storagedriver.vpool_guid) if ovsdb.exists(vpool_config_key): # Second node gets values saved by first node cinder_password, cinder_user, tenant_name, controller_ip, config_cinder = ovsdb.get(vpool_config_key) else: config_cinder = parameters.get('config_cinder', False) cinder_password = '' cinder_user = '' tenant_name = '' controller_ip = '' if config_cinder: cinder_password = parameters.get('cinder_pass', cinder_password) cinder_user = parameters.get('cinder_user', cinder_user) tenant_name = parameters.get('cinder_tenant', tenant_name) controller_ip = parameters.get('cinder_controller', controller_ip) # Keystone host if cinder_password: osc = OpenStackCinder(cinder_password = cinder_password, cinder_user = cinder_user, tenant_name = tenant_name, controller_ip = controller_ip) osc.configure_vpool(vpool_name, storagedriver.mountpoint) # Save values for first node to use ovsdb.set(vpool_config_key, [cinder_password, cinder_user, tenant_name, controller_ip, config_cinder])
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at verison 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 1 introduced: # - The datastore is still empty, add defaults if working_version < 1: from ovs.dal.hybrids.user import User from ovs.dal.hybrids.group import Group from ovs.dal.hybrids.role import Role from ovs.dal.hybrids.client import Client from ovs.dal.hybrids.j_rolegroup import RoleGroup from ovs.dal.hybrids.j_roleclient import RoleClient from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.branding import Branding from ovs.dal.lists.backendtypelist import BackendTypeList # Create groups admin_group = Group() admin_group.name = 'administrators' admin_group.description = 'Administrators' admin_group.save() viewers_group = Group() viewers_group.name = 'viewers' viewers_group.description = 'Viewers' viewers_group.save() # Create users admin = User() admin.username = '******' admin.password = hashlib.sha256('admin').hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = 'INTERNAL' admin_pw_client.grant_type = 'PASSWORD' admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = 'INTERNAL' admin_cc_client.grant_type = 'CLIENT_CREDENTIALS' admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters + string.digits + '|_=+*#@!/-[]{}<>.?,\'";:~') for _ in range(128)) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = 'read' read_role.name = 'Read' read_role.description = 'Can read objects' read_role.save() write_role = Role() write_role.code = 'write' write_role.name = 'Write' write_role.description = 'Can write objects' write_role.save() manage_role = Role() manage_role.code = 'manage' manage_role.name = 'Manage' manage_role.description = 'Can manage the system' manage_role.save() # Attach groups to roles mapping = [ (admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role]) ] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add backends for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'), ('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in ['MetadataServer', 'AlbaProxy', 'Arakoon']: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Brandings branding = Branding() branding.name = 'Default' branding.description = 'Default bootstrap theme' branding.css = 'bootstrap-default.min.css' branding.productname = 'Open vStorage' branding.is_default = True branding.save() slate = Branding() slate.name = 'Slate' slate.description = 'Dark bootstrap theme' slate.css = 'bootstrap-slate.min.css' slate.productname = 'Open vStorage' slate.is_default = False slate.save() # We're now at version 1 working_version = 1 # Version 2 introduced: # - new Descriptor format if working_version < 2: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data', max_elements=-1) for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry] and 'hybrids' in data[entry]['source']: filename = data[entry]['source'] if not filename.startswith('/'): filename = '/opt/OpenvStorage/ovs/dal/{0}'.format(filename) module = imp.load_source(data[entry]['name'], filename) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) # We're now at version 2 working_version = 2 # Version 3 introduced: # - new Descriptor format if working_version < 3: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data', max_elements=-1) for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry]: module = imp.load_source(data[entry]['name'], data[entry]['source']) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) working_version = 3 return working_version
def get(self, request, *args, **kwargs): """ Fetches metadata """ _ = args, kwargs data = { 'authenticated': False, 'authentication_state': None, 'username': None, 'userguid': None, 'roles': [], 'storagerouter_ips': [sr.ip for sr in StorageRouterList.get_storagerouters()], 'versions': list(settings.VERSION), 'plugins': {} } try: # Gather plugin metadata plugins = {} # - Backends. BackendType plugins must set the has_plugin flag on True for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] data['plugins'] = plugins # Gather authorization metadata if 'HTTP_AUTHORIZATION' not in request.META: return HttpResponse, dict( data.items() + {'authentication_state': 'unauthenticated'}.items()) authorization_type, access_token = request.META[ 'HTTP_AUTHORIZATION'].split(' ') if authorization_type != 'Bearer': return HttpResponse, dict( data.items() + {'authentication_state': 'invalid_authorization_type' }.items()) tokens = BearerTokenList.get_by_access_token(access_token) if len(tokens) != 1: return HttpResponse, dict( data.items() + {'authentication_state': 'invalid_token'}.items()) token = tokens[0] if token.expiration < time.time(): for junction in token.roles.itersafe(): junction.delete() token.delete() return HttpResponse, dict( data.items() + {'authentication_state': 'token_expired'}.items()) # Gather user metadata user = token.client.user if not user.is_active: return HttpResponse, dict( data.items() + {'authentication_state': 'inactive_user'}.items()) roles = [j.role.code for j in token.roles] return HttpResponse, dict( data.items() + { 'authenticated': True, 'authentication_state': 'authenticated', 'username': user.username, 'userguid': user.guid, 'roles': roles, 'plugins': plugins }.items()) except Exception as ex: logger.exception('Unexpected exception: {0}'.format(ex)) return HttpResponse, dict( data.items() + {'authentication_state': 'unexpected_exception'}.items())
def migrate(previous_version): """ Migrates from a given version to the current version. It uses 'previous_version' to be smart wherever possible, but the code should be able to migrate any version towards the expected version. When this is not possible, the code can set a minimum version and raise when it is not met. :param previous_version: The previous version from which to start the migration :type previous_version: float """ working_version = previous_version if working_version == 0: # Initial version: # * Set the version to THIS RELEASE version from ovs.dal.hybrids.user import User from ovs.dal.hybrids.group import Group from ovs.dal.hybrids.role import Role from ovs.dal.hybrids.client import Client from ovs.dal.hybrids.j_rolegroup import RoleGroup from ovs.dal.hybrids.j_roleclient import RoleClient from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.branding import Branding from ovs.dal.lists.backendtypelist import BackendTypeList # Create groups admin_group = Group() admin_group.name = 'administrators' admin_group.description = 'Administrators' admin_group.save() viewers_group = Group() viewers_group.name = 'viewers' viewers_group.description = 'Viewers' viewers_group.save() # Create users admin = User() admin.username = '******' admin.password = hashlib.sha256('admin').hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = 'INTERNAL' admin_pw_client.grant_type = 'PASSWORD' admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = 'INTERNAL' admin_cc_client.grant_type = 'CLIENT_CREDENTIALS' admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters + string.digits + '|_=+*#@!/-[]{}<>.?,\'";:~') for _ in range(128)) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = 'read' read_role.name = 'Read' read_role.description = 'Can read objects' read_role.save() write_role = Role() write_role.code = 'write' write_role.name = 'Write' write_role.description = 'Can write objects' write_role.save() manage_role = Role() manage_role.code = 'manage' manage_role.name = 'Manage' manage_role.description = 'Can manage the system' manage_role.save() # Attach groups to roles mapping = [ (admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role]) ] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add service types for service_type_info in [ServiceType.SERVICE_TYPES.MD_SERVER, ServiceType.SERVICE_TYPES.ALBA_PROXY, ServiceType.SERVICE_TYPES.ARAKOON]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Branding branding = Branding() branding.name = 'Default' branding.description = 'Default bootstrap theme' branding.css = 'bootstrap-default.min.css' branding.productname = 'Open vStorage' branding.is_default = True branding.save() slate = Branding() slate.name = 'Slate' slate.description = 'Dark bootstrap theme' slate.css = 'bootstrap-slate.min.css' slate.productname = 'Open vStorage' slate.is_default = False slate.save() # From here on, all actual migration should happen to get to the expected state for THIS RELEASE elif working_version < OVSMigrator.THIS_VERSION: # Migrate unique constraints from ovs.dal.helpers import HybridRunner, Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): cls = Descriptor().load(class_descriptor).get_object() classname = cls.__name__.lower() unique_key = 'ovs_unique_{0}_{{0}}_'.format(classname) uniques = [] # noinspection PyProtectedMember for prop in cls._properties: if prop.unique is True and len([k for k in client.prefix(unique_key.format(prop.name))]) == 0: uniques.append(prop.name) if len(uniques) > 0: prefix = 'ovs_data_{0}_'.format(classname) for key in client.prefix(prefix): data = client.get(key) for property_name in uniques: ukey = '{0}{1}'.format(unique_key.format(property_name), hashlib.sha1(str(data[property_name])).hexdigest()) client.set(ukey, key) # Complete rework of the way we detect devices to assign roles or use as ASD # Allow loop-, raid-, nvme-, ??-devices and logical volumes as ASD (https://github.com/openvstorage/framework/issues/792) from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.extensions.generic.sshclient import SSHClient, UnableToConnectException from ovs.lib.disk import DiskController for storagerouter in StorageRouterList.get_storagerouters(): try: client = SSHClient(storagerouter, username='******') except UnableToConnectException: raise # Retrieve all symlinks for all devices # Example of name_alias_mapping: # {'/dev/md0': ['/dev/disk/by-id/md-uuid-ad2de634:26d97253:5eda0a23:96986b76', '/dev/disk/by-id/md-name-OVS-1:0'], # '/dev/sda': ['/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0'], # '/dev/sda1': ['/dev/disk/by-uuid/e3e0bc62-4edc-4c6b-a6ce-1f39e8f27e41', '/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0-part1']} name_alias_mapping = {} alias_name_mapping = {} for path_type in client.dir_list(directory='/dev/disk'): if path_type in ['by-uuid', 'by-partuuid']: # UUIDs can change after creating a filesystem on a partition continue directory = '/dev/disk/{0}'.format(path_type) for symlink in client.dir_list(directory=directory): symlink_path = '{0}/{1}'.format(directory, symlink) link = client.file_read_link(symlink_path) if link not in name_alias_mapping: name_alias_mapping[link] = [] name_alias_mapping[link].append(symlink_path) alias_name_mapping[symlink_path] = link for disk in storagerouter.disks: if disk.aliases is None: # noinspection PyProtectedMember device_path = '/dev/{0}'.format(disk.name) disk.aliases = name_alias_mapping.get(device_path, [device_path]) disk.save() for partition in disk.partitions: if partition.aliases is None: # noinspection PyProtectedMember partition_device = alias_name_mapping.get(partition._data.get('path')) if partition_device is None: partition.aliases = [] partition.save() continue partition.aliases = name_alias_mapping.get(partition_device, []) partition.save() DiskController.sync_with_reality(storagerouter_guid=storagerouter.guid) # Only support ALBA backend type from ovs.dal.lists.backendtypelist import BackendTypeList for backend_type in BackendTypeList.get_backend_types(): if backend_type.code != 'alba': backend_type.delete() # Reformat the vpool.metadata information from ovs.dal.lists.vpoollist import VPoolList for vpool in VPoolList.get_vpools(): new_metadata = {} for metadata_key, value in vpool.metadata.items(): new_info = {} storagerouter_guids = [key for key in vpool.metadata.keys() if not key.startswith('backend')] if isinstance(value, dict): read_cache = value.get('backend_info', {}).get('fragment_cache_on_read', True) write_cache = value.get('backend_info', {}).get('fragment_cache_on_write', False) new_info['backend_info'] = {'alba_backend_guid': value.get('backend_guid'), 'backend_guid': None, 'frag_size': value.get('backend_info', {}).get('frag_size'), 'name': value.get('name'), 'policies': value.get('backend_info', {}).get('policies'), 'preset': value.get('preset'), 'sco_size': value.get('backend_info', {}).get('sco_size'), 'total_size': value.get('backend_info', {}).get('total_size')} new_info['arakoon_config'] = value.get('arakoon_config') new_info['connection_info'] = {'host': value.get('connection', {}).get('host', ''), 'port': value.get('connection', {}).get('port', ''), 'local': value.get('connection', {}).get('local', ''), 'client_id': value.get('connection', {}).get('client_id', ''), 'client_secret': value.get('connection', {}).get('client_secret', '')} if metadata_key == 'backend': new_info['caching_info'] = dict((sr_guid, {'fragment_cache_on_read': read_cache, 'fragment_cache_on_write': write_cache}) for sr_guid in storagerouter_guids) if metadata_key in storagerouter_guids: metadata_key = 'backend_aa_{0}'.format(metadata_key) new_metadata[metadata_key] = new_info vpool.metadata = new_metadata vpool.save() # Removal of READ role from ovs.dal.lists.diskpartitionlist import DiskPartitionList for partition in DiskPartitionList.get_partitions(): if 'READ' in partition.roles: partition.roles.remove('READ') partition.save() return OVSMigrator.THIS_VERSION
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at verison 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 1 introduced: # - The datastore is still empty, add defaults if working_version < 1: from ovs.dal.hybrids.user import User from ovs.dal.hybrids.group import Group from ovs.dal.hybrids.role import Role from ovs.dal.hybrids.client import Client from ovs.dal.hybrids.j_rolegroup import RoleGroup from ovs.dal.hybrids.j_roleclient import RoleClient from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.branding import Branding from ovs.dal.lists.backendtypelist import BackendTypeList # Create groups admin_group = Group() admin_group.name = 'administrators' admin_group.description = 'Administrators' admin_group.save() viewers_group = Group() viewers_group.name = 'viewers' viewers_group.description = 'Viewers' viewers_group.save() # Create users admin = User() admin.username = '******' admin.password = hashlib.sha256('admin').hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = 'INTERNAL' admin_pw_client.grant_type = 'PASSWORD' admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = 'INTERNAL' admin_cc_client.grant_type = 'CLIENT_CREDENTIALS' admin_cc_client.client_secret = ''.join( random.choice(string.ascii_letters + string.digits + '|_=+*#@!/-[]{}<>.?,\'";:~') for _ in range(128)) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = 'read' read_role.name = 'Read' read_role.description = 'Can read objects' read_role.save() write_role = Role() write_role.code = 'write' write_role.name = 'Write' write_role.description = 'Can write objects' write_role.save() manage_role = Role() manage_role.code = 'manage' manage_role.name = 'Manage' manage_role.description = 'Can manage the system' manage_role.save() # Attach groups to roles mapping = [(admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role])] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add backends for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'), ('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in [ 'MetadataServer', 'AlbaProxy', 'Arakoon' ]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Brandings branding = Branding() branding.name = 'Default' branding.description = 'Default bootstrap theme' branding.css = 'bootstrap-default.min.css' branding.productname = 'Open vStorage' branding.is_default = True branding.save() slate = Branding() slate.name = 'Slate' slate.description = 'Dark bootstrap theme' slate.css = 'bootstrap-slate.min.css' slate.productname = 'Open vStorage' slate.is_default = False slate.save() # We're now at version 1 working_version = 1 # Version 2 introduced: # - new Descriptor format if working_version < 2: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data', max_elements=-1) for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[ entry] and 'hybrids' in data[entry]['source']: filename = data[entry]['source'] if not filename.startswith('/'): filename = '/opt/OpenvStorage/ovs/dal/{0}'.format( filename) module = imp.load_source(data[entry]['name'], filename) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) # We're now at version 2 working_version = 2 # Version 3 introduced: # - new Descriptor format if working_version < 3: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data', max_elements=-1) for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry]: module = imp.load_source(data[entry]['name'], data[entry]['source']) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) working_version = 3 return working_version
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at verison 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 0.0.1 introduced: if working_version < 1: # Create groups admin_group = Group() admin_group.name = "administrators" admin_group.description = "Administrators" admin_group.save() viewers_group = Group() viewers_group.name = "viewers" viewers_group.description = "Viewers" viewers_group.save() # Create users admin = User() admin.username = "******" admin.password = hashlib.sha256("admin").hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = "INTERNAL" admin_pw_client.grant_type = "PASSWORD" admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = "INTERNAL" admin_cc_client.grant_type = "CLIENT_CREDENTIALS" admin_cc_client.client_secret = "".join( random.choice(string.ascii_letters + string.digits + "|_=+*#@!/-[]{}<>.?,'\";:~") for _ in range(128) ) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = "read" read_role.name = "Read" read_role.description = "Can read objects" read_role.save() write_role = Role() write_role.code = "write" write_role.name = "Write" write_role.description = "Can write objects" write_role.save() manage_role = Role() manage_role.code = "manage" manage_role.name = "Manage" manage_role.description = "Can manage the system" manage_role.save() # Attach groups to roles mapping = [(admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role])] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add backends for backend_type_info in [ ("Ceph", "ceph_s3"), ("Amazon", "amazon_s3"), ("Swift", "swift_s3"), ("Local", "local"), ("Distributed", "distributed"), ("ALBA", "alba"), ]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in ["MetadataServer", "AlbaProxy"]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Brandings branding = Branding() branding.name = "Default" branding.description = "Default bootstrap theme" branding.css = "bootstrap-default.min.css" branding.productname = "Open vStorage" branding.is_default = True branding.save() slate = Branding() slate.name = "Slate" slate.description = "Dark bootstrap theme" slate.css = "bootstrap-slate.min.css" slate.productname = "Open vStorage" slate.is_default = False slate.save() # We're now at version 0.0.1 working_version = 1 # Version 0.0.2 introduced: if working_version < 2: # Execute some code that upgrades to version 2 # working_version = 2 pass return working_version
def get_valid_backendtypes(): """ Retrieve a list of supported Backend Types :return: List of Backend Type Names """ return [backend_type.code for backend_type in BackendTypeList.get_backend_types()]
def get(self, request, *args, **kwargs): """ Fetches metadata """ _ = args, kwargs data = { 'authenticated': False, 'authentication_state': None, 'authentication_metadata': {}, 'username': None, 'userguid': None, 'roles': [], 'identification': {}, 'storagerouter_ips': [sr.ip for sr in StorageRouterList.get_storagerouters()], 'versions': list(settings.VERSION), 'plugins': {} } try: # Gather plugin metadata plugins = {} # - Backends. BackendType plugins must set the has_plugin flag on True for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] # - Generic plugins, as added to the configuration file(s) generic_plugins = EtcdConfiguration.get( '/ovs/framework/plugins/installed|generic') for plugin_name in generic_plugins: if plugin_name not in plugins: plugins[plugin_name] = [] plugins[plugin_name] += ['gui'] data['plugins'] = plugins # Fill identification data['identification'] = { 'cluster_id': EtcdConfiguration.get('/ovs/framework/cluster_id') } # Get authentication metadata authentication_metadata = {'ip': System.get_my_storagerouter().ip} for key in ['mode', 'authorize_uri', 'client_id', 'scope']: if EtcdConfiguration.exists( '/ovs/framework/webapps|oauth2.{0}'.format(key)): authentication_metadata[key] = EtcdConfiguration.get( '/ovs/framework/webapps|oauth2.{0}'.format(key)) data['authentication_metadata'] = authentication_metadata # Gather authorization metadata if 'HTTP_AUTHORIZATION' not in request.META: return HttpResponse, dict( data.items() + {'authentication_state': 'unauthenticated'}.items()) authorization_type, access_token = request.META[ 'HTTP_AUTHORIZATION'].split(' ') if authorization_type != 'Bearer': return HttpResponse, dict( data.items() + {'authentication_state': 'invalid_authorization_type' }.items()) tokens = BearerTokenList.get_by_access_token(access_token) if len(tokens) != 1: return HttpResponse, dict( data.items() + {'authentication_state': 'invalid_token'}.items()) token = tokens[0] if token.expiration < time.time(): for junction in token.roles.itersafe(): junction.delete() token.delete() return HttpResponse, dict( data.items() + {'authentication_state': 'token_expired'}.items()) # Gather user metadata user = token.client.user if not user.is_active: return HttpResponse, dict( data.items() + {'authentication_state': 'inactive_user'}.items()) roles = [j.role.code for j in token.roles] return HttpResponse, dict( data.items() + { 'authenticated': True, 'authentication_state': 'authenticated', 'username': user.username, 'userguid': user.guid, 'roles': roles, 'plugins': plugins }.items()) except Exception as ex: MetadataView._logger.exception( 'Unexpected exception: {0}'.format(ex)) return HttpResponse, dict( data.items() + {'authentication_state': 'unexpected_exception'}.items())
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at version 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 1 introduced: # - The datastore is still empty, add defaults if working_version < 1: from ovs.dal.hybrids.user import User from ovs.dal.hybrids.group import Group from ovs.dal.hybrids.role import Role from ovs.dal.hybrids.client import Client from ovs.dal.hybrids.failuredomain import FailureDomain from ovs.dal.hybrids.j_rolegroup import RoleGroup from ovs.dal.hybrids.j_roleclient import RoleClient from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.branding import Branding from ovs.dal.lists.backendtypelist import BackendTypeList # Create groups admin_group = Group() admin_group.name = 'administrators' admin_group.description = 'Administrators' admin_group.save() viewers_group = Group() viewers_group.name = 'viewers' viewers_group.description = 'Viewers' viewers_group.save() # Create users admin = User() admin.username = '******' admin.password = hashlib.sha256('admin').hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = 'INTERNAL' admin_pw_client.grant_type = 'PASSWORD' admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = 'INTERNAL' admin_cc_client.grant_type = 'CLIENT_CREDENTIALS' admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters + string.digits + '|_=+*#@!/-[]{}<>.?,\'";:~') for _ in range(128)) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = 'read' read_role.name = 'Read' read_role.description = 'Can read objects' read_role.save() write_role = Role() write_role.code = 'write' write_role.name = 'Write' write_role.description = 'Can write objects' write_role.save() manage_role = Role() manage_role.code = 'manage' manage_role.name = 'Manage' manage_role.description = 'Can manage the system' manage_role.save() # Attach groups to roles mapping = [ (admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role]) ] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add backends for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'), ('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in [ServiceType.SERVICE_TYPES.MD_SERVER, ServiceType.SERVICE_TYPES.ALBA_PROXY, ServiceType.SERVICE_TYPES.ARAKOON]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Branding branding = Branding() branding.name = 'Default' branding.description = 'Default bootstrap theme' branding.css = 'bootstrap-default.min.css' branding.productname = 'Open vStorage' branding.is_default = True branding.save() slate = Branding() slate.name = 'Slate' slate.description = 'Dark bootstrap theme' slate.css = 'bootstrap-slate.min.css' slate.productname = 'Open vStorage' slate.is_default = False slate.save() # Failure Domain failure_domain = FailureDomain() failure_domain.name = 'Default' failure_domain.save() # We're now at version 1 working_version = 1 # Version 2 introduced: # - new Descriptor format if working_version < 2: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data') for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry] and 'hybrids' in data[entry]['source']: filename = data[entry]['source'] if not filename.startswith('/'): filename = '/opt/OpenvStorage/ovs/dal/{0}'.format(filename) module = imp.load_source(data[entry]['name'], filename) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) # We're now at version 2 working_version = 2 # Version 3 introduced: # - new Descriptor format if working_version < 3: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data') for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry]: module = imp.load_source(data[entry]['name'], data[entry]['source']) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) working_version = 3 # Version 4 introduced: # - Flexible SSD layout if working_version < 4: import os from ovs.dal.hybrids.diskpartition import DiskPartition from ovs.dal.hybrids.j_storagedriverpartition import StorageDriverPartition from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.lists.servicetypelist import ServiceTypeList from ovs.dal.lists.storagedriverlist import StorageDriverList from ovs.extensions.generic.remote import remote from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration for service in ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.MD_SERVER).services: mds_service = service.mds_service storagedriver = None for current_storagedriver in service.storagerouter.storagedrivers: if current_storagedriver.vpool_guid == mds_service.vpool_guid: storagedriver = current_storagedriver break tasks = {} if storagedriver._data.get('mountpoint_md'): tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_md'), storagedriver.vpool.name, mds_service.number)] = (DiskPartition.ROLES.DB, StorageDriverPartition.SUBROLE.MDS) if storagedriver._data.get('mountpoint_temp'): tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_temp'), storagedriver.vpool.name, mds_service.number)] = (DiskPartition.ROLES.SCRUB, StorageDriverPartition.SUBROLE.MDS) for disk in service.storagerouter.disks: for partition in disk.partitions: for directory, (role, subrole) in tasks.iteritems(): with remote(storagedriver.storagerouter.ip, [os], username='******') as rem: stat_dir = directory while not rem.os.path.exists(stat_dir) and stat_dir != '/': stat_dir = stat_dir.rsplit('/', 1)[0] if not stat_dir: stat_dir = '/' inode = rem.os.stat(stat_dir).st_dev if partition.inode == inode: if role not in partition.roles: partition.roles.append(role) partition.save() number = 0 migrated = False for sd_partition in storagedriver.partitions: if sd_partition.role == role and sd_partition.sub_role == subrole: if sd_partition.mds_service == mds_service: migrated = True break if sd_partition.partition_guid == partition.guid: number = max(sd_partition.number, number) if migrated is False: sd_partition = StorageDriverPartition() sd_partition.role = role sd_partition.sub_role = subrole sd_partition.partition = partition sd_partition.storagedriver = storagedriver sd_partition.mds_service = mds_service sd_partition.size = None sd_partition.number = number + 1 sd_partition.save() client = SSHClient(storagedriver.storagerouter, username='******') path = sd_partition.path.rsplit('/', 1)[0] if path: client.dir_create(path) client.dir_chown(path, 'ovs', 'ovs') client.dir_create(directory) client.dir_chown(directory, 'ovs', 'ovs') client.symlink({sd_partition.path: directory}) for storagedriver in StorageDriverList.get_storagedrivers(): migrated_objects = {} for disk in storagedriver.storagerouter.disks: for partition in disk.partitions: # Process all mountpoints that are unique and don't have a specified size for key, (role, sr_info) in {'mountpoint_md': (DiskPartition.ROLES.DB, {'metadata_{0}': StorageDriverPartition.SUBROLE.MD, 'tlogs_{0}': StorageDriverPartition.SUBROLE.TLOG}), 'mountpoint_fragmentcache': (DiskPartition.ROLES.WRITE, {'fcache_{0}': StorageDriverPartition.SUBROLE.FCACHE}), 'mountpoint_foc': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD, 'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}), 'mountpoint_dtl': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD, 'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}), 'mountpoint_readcaches': (DiskPartition.ROLES.READ, {'': None}), 'mountpoint_writecaches': (DiskPartition.ROLES.WRITE, {'sco_{0}': StorageDriverPartition.SUBROLE.SCO})}.iteritems(): if key in storagedriver._data: is_list = isinstance(storagedriver._data[key], list) entries = storagedriver._data[key][:] if is_list is True else [storagedriver._data[key]] for entry in entries: if not entry: if is_list: storagedriver._data[key].remove(entry) if len(storagedriver._data[key]) == 0: del storagedriver._data[key] else: del storagedriver._data[key] else: with remote(storagedriver.storagerouter.ip, [os], username='******') as rem: inode = rem.os.stat(entry).st_dev if partition.inode == inode: if role not in partition.roles: partition.roles.append(role) partition.save() for folder, subrole in sr_info.iteritems(): number = 0 migrated = False for sd_partition in storagedriver.partitions: if sd_partition.role == role and sd_partition.sub_role == subrole: if sd_partition.partition_guid == partition.guid: number = max(sd_partition.number, number) if migrated is False: sd_partition = StorageDriverPartition() sd_partition.role = role sd_partition.sub_role = subrole sd_partition.partition = partition sd_partition.storagedriver = storagedriver sd_partition.size = None sd_partition.number = number + 1 sd_partition.save() if folder: source = '{0}/{1}'.format(entry, folder.format(storagedriver.vpool.name)) else: source = entry client = SSHClient(storagedriver.storagerouter, username='******') path = sd_partition.path.rsplit('/', 1)[0] if path: client.dir_create(path) client.dir_chown(path, 'ovs', 'ovs') client.symlink({sd_partition.path: source}) migrated_objects[source] = sd_partition if is_list: storagedriver._data[key].remove(entry) if len(storagedriver._data[key]) == 0: del storagedriver._data[key] else: del storagedriver._data[key] storagedriver.save() if 'mountpoint_bfs' in storagedriver._data: storagedriver.mountpoint_dfs = storagedriver._data['mountpoint_bfs'] if not storagedriver.mountpoint_dfs: storagedriver.mountpoint_dfs = None del storagedriver._data['mountpoint_bfs'] storagedriver.save() if 'mountpoint_temp' in storagedriver._data: del storagedriver._data['mountpoint_temp'] storagedriver.save() if migrated_objects: print 'Loading sizes' config = StorageDriverConfiguration('storagedriver', storagedriver.vpool_guid, storagedriver.storagedriver_id) config.load() for readcache in config.configuration.get('content_addressed_cache', {}).get('clustercache_mount_points', []): path = readcache.get('path', '').rsplit('/', 1)[0] size = int(readcache['size'].strip('KiB')) * 1024 if 'size' in readcache else None if path in migrated_objects: migrated_objects[path].size = long(size) migrated_objects[path].save() for writecache in config.configuration.get('scocache', {}).get('scocache_mount_points', []): path = writecache.get('path', '') size = int(writecache['size'].strip('KiB')) * 1024 if 'size' in writecache else None if path in migrated_objects: migrated_objects[path].size = long(size) migrated_objects[path].save() working_version = 4 # Version 5 introduced: # - Failure Domains if working_version < 5: import os from ovs.dal.hybrids.failuredomain import FailureDomain from ovs.dal.lists.failuredomainlist import FailureDomainList from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.extensions.generic.remote import remote from ovs.extensions.generic.sshclient import SSHClient failure_domains = FailureDomainList.get_failure_domains() if len(failure_domains) > 0: failure_domain = failure_domains[0] else: failure_domain = FailureDomain() failure_domain.name = 'Default' failure_domain.save() for storagerouter in StorageRouterList.get_storagerouters(): change = False if storagerouter.primary_failure_domain is None: storagerouter.primary_failure_domain = failure_domain change = True if storagerouter.rdma_capable is None: client = SSHClient(storagerouter, username='******') rdma_capable = False with remote(client.ip, [os], username='******') as rem: for root, dirs, files in rem.os.walk('/sys/class/infiniband'): for directory in dirs: ports_dir = '/'.join([root, directory, 'ports']) if not rem.os.path.exists(ports_dir): continue for sub_root, sub_dirs, _ in rem.os.walk(ports_dir): if sub_root != ports_dir: continue for sub_directory in sub_dirs: state_file = '/'.join([sub_root, sub_directory, 'state']) if rem.os.path.exists(state_file): if 'ACTIVE' in client.run('cat {0}'.format(state_file)): rdma_capable = True storagerouter.rdma_capable = rdma_capable change = True if change is True: storagerouter.save() working_version = 5 # Version 6 introduced: # - Distributed scrubbing if working_version < 6: from ovs.dal.hybrids.diskpartition import DiskPartition from ovs.dal.lists.storagedriverlist import StorageDriverList from ovs.extensions.generic.sshclient import SSHClient for storage_driver in StorageDriverList.get_storagedrivers(): root_client = SSHClient(storage_driver.storagerouter, username='******') for partition in storage_driver.partitions: if partition.role == DiskPartition.ROLES.SCRUB: old_path = partition.path partition.sub_role = None partition.save() partition.invalidate_dynamics(['folder', 'path']) if root_client.dir_exists(partition.path): continue # New directory already exists if '_mds_' in old_path: if root_client.dir_exists(old_path): root_client.symlink({partition.path: old_path}) if not root_client.dir_exists(partition.path): root_client.dir_create(partition.path) root_client.dir_chmod(partition.path, 0777) working_version = 6 # Version 7 introduced: # - vPool status if working_version < 7: from ovs.dal.hybrids import vpool reload(vpool) from ovs.dal.hybrids.vpool import VPool from ovs.dal.lists.vpoollist import VPoolList for _vpool in VPoolList.get_vpools(): vpool = VPool(_vpool.guid) if hasattr(vpool, 'status') and vpool.status is None: vpool.status = VPool.STATUSES.RUNNING vpool.save() working_version = 7 # Version 10 introduced: # - Reverse indexes are stored in persistent store # - Store more non-changing metadata on disk iso using a dynamic property if working_version < 10: from ovs.dal.helpers import HybridRunner, Descriptor from ovs.dal.datalist import DataList from ovs.extensions.storage.persistentfactory import PersistentFactory from ovs.extensions.storage.volatilefactory import VolatileFactory persistent = PersistentFactory.get_client() for prefix in ['ovs_listcache', 'ovs_reverseindex']: for key in persistent.prefix(prefix): persistent.delete(key) for key in persistent.prefix('ovs_data_'): persistent.set(key, persistent.get(key)) base_reverse_key = 'ovs_reverseindex_{0}_{1}|{2}|{3}' hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): cls = Descriptor().load(class_descriptor).get_object() all_objects = DataList(cls, {'type': DataList.where_operator.AND, 'items': []}) for item in all_objects: guid = item.guid for relation in item._relations: if relation.foreign_type is None: rcls = cls rclsname = rcls.__name__.lower() else: rcls = relation.foreign_type rclsname = rcls.__name__.lower() key = relation.name rguid = item._data[key]['guid'] if rguid is not None: reverse_key = base_reverse_key.format(rclsname, rguid, relation.foreign_key, guid) persistent.set(reverse_key, 0) volatile = VolatileFactory.get_client() try: volatile._client.flush_all() except: pass from ovs.dal.lists.vdisklist import VDiskList for vdisk in VDiskList.get_vdisks(): try: vdisk.metadata = {'lba_size': vdisk.info['lba_size'], 'cluster_multiplier': vdisk.info['cluster_multiplier']} vdisk.save() except: pass working_version = 10 # Version 11 introduced: # - ALBA accelerated ALBA, meaning different vpool.metadata information if working_version < 11: from ovs.dal.lists.vpoollist import VPoolList for vpool in VPoolList.get_vpools(): vpool.metadata = {'backend': vpool.metadata} if 'metadata' in vpool.metadata['backend']: vpool.metadata['backend']['arakoon_config'] = vpool.metadata['backend'].pop('metadata') if 'backend_info' in vpool.metadata['backend']: vpool.metadata['backend']['backend_info']['fragment_cache_on_read'] = True vpool.metadata['backend']['backend_info']['fragment_cache_on_write'] = False vpool.save() working_version = 11 return working_version
def get(self, request, *args, **kwargs): """ Fetches metadata """ _ = args, kwargs data = {'authenticated': False, 'authentication_state': None, 'authentication_metadata': {}, 'username': None, 'userguid': None, 'roles': [], 'identification': {}, 'storagerouter_ips': [sr.ip for sr in StorageRouterList.get_storagerouters()], 'versions': list(settings.VERSION), 'plugins': {}} try: # Gather plugin metadata plugins = {} # - Backends. BackendType plugins must set the has_plugin flag on True for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] # - Generic plugins, as added to the configuration file(s) generic_plugins = Configuration.get('/ovs/framework/plugins/installed|generic') for plugin_name in generic_plugins: if plugin_name not in plugins: plugins[plugin_name] = [] plugins[plugin_name] += ['gui'] data['plugins'] = plugins # Fill identification data['identification'] = {'cluster_id': Configuration.get('/ovs/framework/cluster_id')} # Get authentication metadata authentication_metadata = {'ip': System.get_my_storagerouter().ip} for key in ['mode', 'authorize_uri', 'client_id', 'scope']: if Configuration.exists('/ovs/framework/webapps|oauth2.{0}'.format(key)): authentication_metadata[key] = Configuration.get('/ovs/framework/webapps|oauth2.{0}'.format(key)) data['authentication_metadata'] = authentication_metadata # Gather authorization metadata if 'HTTP_AUTHORIZATION' not in request.META: return dict(data.items() + {'authentication_state': 'unauthenticated'}.items()) authorization_type, access_token = request.META['HTTP_AUTHORIZATION'].split(' ') if authorization_type != 'Bearer': return dict(data.items() + {'authentication_state': 'invalid_authorization_type'}.items()) tokens = BearerTokenList.get_by_access_token(access_token) if len(tokens) != 1: return dict(data.items() + {'authentication_state': 'invalid_token'}.items()) token = tokens[0] if token.expiration < time.time(): for junction in token.roles.itersafe(): junction.delete() token.delete() return dict(data.items() + {'authentication_state': 'token_expired'}.items()) # Gather user metadata user = token.client.user if not user.is_active: return dict(data.items() + {'authentication_state': 'inactive_user'}.items()) roles = [j.role.code for j in token.roles] return dict(data.items() + {'authenticated': True, 'authentication_state': 'authenticated', 'username': user.username, 'userguid': user.guid, 'roles': roles, 'plugins': plugins}.items()) except Exception as ex: MetadataView._logger.exception('Unexpected exception: {0}'.format(ex)) return dict(data.items() + {'authentication_state': 'unexpected_exception'}.items())
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at verison 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 1 introduced: # - The datastore is still empty, add defaults if working_version < 1: from ovs.dal.hybrids.user import User from ovs.dal.hybrids.group import Group from ovs.dal.hybrids.role import Role from ovs.dal.hybrids.client import Client from ovs.dal.hybrids.j_rolegroup import RoleGroup from ovs.dal.hybrids.j_roleclient import RoleClient from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.branding import Branding from ovs.dal.lists.backendtypelist import BackendTypeList # Create groups admin_group = Group() admin_group.name = 'administrators' admin_group.description = 'Administrators' admin_group.save() viewers_group = Group() viewers_group.name = 'viewers' viewers_group.description = 'Viewers' viewers_group.save() # Create users admin = User() admin.username = '******' admin.password = hashlib.sha256('admin').hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = 'INTERNAL' admin_pw_client.grant_type = 'PASSWORD' admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = 'INTERNAL' admin_cc_client.grant_type = 'CLIENT_CREDENTIALS' admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters + string.digits + '|_=+*#@!/-[]{}<>.?,\'";:~') for _ in range(128)) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = 'read' read_role.name = 'Read' read_role.description = 'Can read objects' read_role.save() write_role = Role() write_role.code = 'write' write_role.name = 'Write' write_role.description = 'Can write objects' write_role.save() manage_role = Role() manage_role.code = 'manage' manage_role.name = 'Manage' manage_role.description = 'Can manage the system' manage_role.save() # Attach groups to roles mapping = [ (admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role]) ] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add backends for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'), ('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in ['MetadataServer', 'AlbaProxy', 'Arakoon']: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Brandings branding = Branding() branding.name = 'Default' branding.description = 'Default bootstrap theme' branding.css = 'bootstrap-default.min.css' branding.productname = 'Open vStorage' branding.is_default = True branding.save() slate = Branding() slate.name = 'Slate' slate.description = 'Dark bootstrap theme' slate.css = 'bootstrap-slate.min.css' slate.productname = 'Open vStorage' slate.is_default = False slate.save() # We're now at version 1 working_version = 1 # Version 2 introduced: # - new Descriptor format if working_version < 2: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data') for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry] and 'hybrids' in data[entry]['source']: filename = data[entry]['source'] if not filename.startswith('/'): filename = '/opt/OpenvStorage/ovs/dal/{0}'.format(filename) module = imp.load_source(data[entry]['name'], filename) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) # We're now at version 2 working_version = 2 # Version 3 introduced: # - new Descriptor format if working_version < 3: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data') for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry]: module = imp.load_source(data[entry]['name'], data[entry]['source']) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) working_version = 3 # Version 4 introduced: # - Flexible SSD layout if working_version < 4: import os from ovs.dal.lists.storagedriverlist import StorageDriverList from ovs.dal.hybrids.j_storagedriverpartition import StorageDriverPartition from ovs.dal.hybrids.diskpartition import DiskPartition from ovs.dal.lists.servicetypelist import ServiceTypeList from ovs.extensions.generic.remote import Remote from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration for service in ServiceTypeList.get_by_name('MetadataServer').services: mds_service = service.mds_service storagedriver = None for current_storagedriver in service.storagerouter.storagedrivers: if current_storagedriver.vpool_guid == mds_service.vpool_guid: storagedriver = current_storagedriver break tasks = {} if storagedriver._data.get('mountpoint_md'): tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_md'), storagedriver.vpool.name, mds_service.number)] = (DiskPartition.ROLES.DB, StorageDriverPartition.SUBROLE.MDS) if storagedriver._data.get('mountpoint_temp'): tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_temp'), storagedriver.vpool.name, mds_service.number)] = (DiskPartition.ROLES.SCRUB, StorageDriverPartition.SUBROLE.MDS) for disk in service.storagerouter.disks: for partition in disk.partitions: for directory, (role, subrole) in tasks.iteritems(): with Remote(storagedriver.storagerouter.ip, [os], username='******') as remote: stat_dir = directory while not remote.os.path.exists(stat_dir) and stat_dir != '/': stat_dir = stat_dir.rsplit('/', 1)[0] if not stat_dir: stat_dir = '/' inode = remote.os.stat(stat_dir).st_dev if partition.inode == inode: if role not in partition.roles: partition.roles.append(role) partition.save() number = 0 migrated = False for sd_partition in storagedriver.partitions: if sd_partition.role == role and sd_partition.sub_role == subrole: if sd_partition.mds_service == mds_service: migrated = True break if sd_partition.partition_guid == partition.guid: number = max(sd_partition.number, number) if migrated is False: sd_partition = StorageDriverPartition() sd_partition.role = role sd_partition.sub_role = subrole sd_partition.partition = partition sd_partition.storagedriver = storagedriver sd_partition.mds_service = mds_service sd_partition.size = None sd_partition.number = number + 1 sd_partition.save() client = SSHClient(storagedriver.storagerouter, username='******') path = sd_partition.path.rsplit('/', 1)[0] if path: client.dir_create(path) client.dir_chown(path, 'ovs', 'ovs') client.dir_create(directory) client.dir_chown(directory, 'ovs', 'ovs') client.symlink({sd_partition.path: directory}) for storagedriver in StorageDriverList.get_storagedrivers(): migrated_objects = {} for disk in storagedriver.storagerouter.disks: for partition in disk.partitions: # Process all mountpoints that are unique and don't have a specified size for key, (role, sr_info) in {'mountpoint_md': (DiskPartition.ROLES.DB, {'metadata_{0}': StorageDriverPartition.SUBROLE.MD, 'tlogs_{0}': StorageDriverPartition.SUBROLE.TLOG}), 'mountpoint_fragmentcache': (DiskPartition.ROLES.WRITE, {'fcache_{0}': StorageDriverPartition.SUBROLE.FCACHE}), 'mountpoint_foc': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD, 'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}), 'mountpoint_dtl': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD, 'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}), 'mountpoint_readcaches': (DiskPartition.ROLES.READ, {'': None}), 'mountpoint_writecaches': (DiskPartition.ROLES.WRITE, {'sco_{0}': StorageDriverPartition.SUBROLE.SCO})}.iteritems(): if key in storagedriver._data: is_list = isinstance(storagedriver._data[key], list) entries = storagedriver._data[key][:] if is_list is True else [storagedriver._data[key]] for entry in entries: if not entry: if is_list: storagedriver._data[key].remove(entry) if len(storagedriver._data[key]) == 0: del storagedriver._data[key] else: del storagedriver._data[key] else: with Remote(storagedriver.storagerouter.ip, [os], username='******') as remote: inode = remote.os.stat(entry).st_dev if partition.inode == inode: if role not in partition.roles: partition.roles.append(role) partition.save() for folder, subrole in sr_info.iteritems(): number = 0 migrated = False for sd_partition in storagedriver.partitions: if sd_partition.role == role and sd_partition.sub_role == subrole: if sd_partition.partition_guid == partition.guid: number = max(sd_partition.number, number) if migrated is False: sd_partition = StorageDriverPartition() sd_partition.role = role sd_partition.sub_role = subrole sd_partition.partition = partition sd_partition.storagedriver = storagedriver sd_partition.size = None sd_partition.number = number + 1 sd_partition.save() if folder: source = '{0}/{1}'.format(entry, folder.format(storagedriver.vpool.name)) else: source = entry client = SSHClient(storagedriver.storagerouter, username='******') path = sd_partition.path.rsplit('/', 1)[0] if path: client.dir_create(path) client.dir_chown(path, 'ovs', 'ovs') client.symlink({sd_partition.path: source}) migrated_objects[source] = sd_partition if is_list: storagedriver._data[key].remove(entry) if len(storagedriver._data[key]) == 0: del storagedriver._data[key] else: del storagedriver._data[key] storagedriver.save() if 'mountpoint_bfs' in storagedriver._data: storagedriver.mountpoint_dfs = storagedriver._data['mountpoint_bfs'] if not storagedriver.mountpoint_dfs: storagedriver.mountpoint_dfs = None del storagedriver._data['mountpoint_bfs'] storagedriver.save() if 'mountpoint_temp' in storagedriver._data: del storagedriver._data['mountpoint_temp'] storagedriver.save() if migrated_objects: print 'Loading sizes' config = StorageDriverConfiguration('storagedriver', storagedriver.vpool.name) config.load(SSHClient(storagedriver.storagerouter, username='******')) for readcache in config.configuration.get('content_addressed_cache', {}).get('clustercache_mount_points', []): path = readcache.get('path', '').rsplit('/', 1)[0] size = int(readcache['size'].strip('KiB')) * 1024 if 'size' in readcache else None if path in migrated_objects: migrated_objects[path].size = long(size) migrated_objects[path].save() for writecache in config.configuration.get('scocache', {}).get('scocache_mount_points', []): path = writecache.get('path', '') size = int(writecache['size'].strip('KiB')) * 1024 if 'size' in writecache else None if path in migrated_objects: migrated_objects[path].size = long(size) migrated_objects[path].save() working_version = 4 return working_version
def get(self): """ returns OpenAPI specs """ version = settings.VERSION[-1] data = {'swagger': '2.0', 'info': {'title': 'Open vStorage', 'description': 'The Open vStorage API.', 'version': str(version)}, 'basePath': '/api', 'schemes': ['https'], 'consumes': ['application/json'], 'produces': ['application/json; version={0}'.format(version)], 'paths': {'/': {'get': {'summary': 'Retrieve API metadata', 'operationId': 'api', 'responses': {'200': {'descirption': 'API metadata', 'schema': {'type': 'object', 'title': 'APIMetadata', 'properties': {'authenticated': {'type': 'boolean', 'description': 'Indicates whether the client is authenticated.'}, 'authentication_state': {'type': 'string', 'description': 'Povides more information on the "authenticated" state of a client.', 'enum': ['unauthenticated', 'invalid_authorization_type', 'invalid_token', 'token_expired', 'inactive_user', 'authenticated', 'unexpected_exception']}, 'authentication_metadata': {'type': 'object', 'title': 'AuthenticationMetadata', 'description': 'Contains information on the usage of an optional 3rd party OAuth2.0 authentication service.', 'properties': {'ip': {'type': 'string', 'description': 'The IP address of the current node.'}, 'mode': {'type': 'string', 'description': 'Indicates wheter the "local" or a "remote" authentication endpoint should be used.', 'enum': ['local', 'remote']}, 'authorize_uri': {'type': 'string', 'description': 'The URI to which the user has to be redirect to authenticate.'}, 'client_id': {'type': 'string', 'description': 'The client identifier to be used when authenticating.'}, 'scope': {'type': 'string', 'description': 'The scope that has to be requested to the authentication endpoint.'}}, 'required': []}, 'username': {'type': 'string', 'description': 'The username of the client or null if not available.'}, 'userguid': {'type': 'string', 'description': 'The GUID (primary key) of the client\'s user or null if not available.'}, 'roles': {'type': 'array', 'description': 'An array of the scopes that were granted to the client.', 'items': {'type': 'string'}}, 'identification': {'type': 'object', 'title': 'APIIdentification', 'description': 'Contains identification information about the API/environment.', 'properties': {'cluster_id': {'type': 'string', 'description': 'Environment identification string.'}}, 'required': ['cluster_id']}, 'storagerouter_ips': {'type': 'array', 'description': 'An array containing the IP addresses of all StorageRouters in the environment.', 'items': {'type': 'string'}}, 'versions': {'type': 'array', 'description': 'An array of all versions that this instance of the API supports.', 'items': {'type': 'integer'}}, 'plugins': {}}, 'required': ['authenticated', 'authentication_state', 'authentication_metadata', 'username', 'userguid', 'roles', 'identification', 'storagerouter_ips', 'versions', 'plugins']}}}}}}, 'definitions': {'APIError': {'type': 'object', 'properties': {'error': {'type': 'string', 'description': 'An error code'}, 'error_description': {'type': 'string', 'description': 'Descriptive error message'}}, 'required': ['error', 'error_description']}}, 'securityDefinitions': {'oauth2': {'type': 'oauth2', 'flow': 'password', 'tokenUrl': 'oauth2/token', 'scopes': {'read': 'Read access', 'write': 'Write access', 'manage': 'Management access'}}}, 'security': [{'oauth2': ['read', 'write', 'manage']}]} # Plugin information plugins = {} for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] generic_plugins = Configuration.get('/ovs/framework/plugins/installed|generic') for plugin_name in generic_plugins: if plugin_name not in plugins: plugins[plugin_name] = [] plugins[plugin_name] += ['gui'] data['paths']['/']['get']['responses']['200']['schema']['properties']['plugins'] = { 'type': 'object', 'title': 'PluginMetadata', 'description': 'Contains information about plugins active in the system. Each property represents a plugin and the area where they provide functionality.', 'properties': {plugin: {'type': 'array', 'description': 'An array of all areas the plugin provides functionality.', 'items': {'type': 'string'}} for (plugin, info) in plugins.iteritems()}, 'required': [] } # API paths def load_parameters(_fun): # Parameters by @load decorators parameter_info = [] mandatory_args = _fun.ovs_metadata['load']['mandatory'] optional_args = _fun.ovs_metadata['load']['optional'] object_type = _fun.ovs_metadata['load']['object_type'] entries = ['version', 'request', 'local_storagerouter', 'pk', 'contents'] if object_type is not None: object_arg = object_type.__name__.lower() if object_arg in mandatory_args or object_arg in optional_args: parameter_info.append({'name': 'guid', 'in': 'path', 'description': 'Identifier of the object on which to call is applied.', 'required': True, 'type': 'string'}) entries.append(object_arg) for entry in entries: if entry in mandatory_args: mandatory_args.remove(entry) if entry in optional_args: optional_args.remove(entry) docs = _fun.__doc__ doc_info = {} if docs is not None: for match in re.finditer(':(param|type) (.*?): (.*)', docs, re.MULTILINE): entries = match.groups() if entries[1] not in doc_info: doc_info[entries[1]] = {} doc_info[entries[1]][entries[0]] = entries[2] for argument in mandatory_args + optional_args: info = {'name': argument, 'in': 'query', 'required': argument in mandatory_args, 'type': 'string'} if argument in doc_info: description = doc_info[argument].get('param') if description: info['description'] = description type_info = doc_info[argument].get('type') if type_info: if type_info in ['int', 'long']: info['type'] = 'integer' elif type_info in ['float']: info['type'] = 'number' elif type_info in ['bool']: info['type'] = 'boolean' elif type_info in ['str', 'basestring', 'unicode']: info['type'] = 'string' elif type_info in ['dict']: info['type'] = 'object' parameter_info.append(info) # Parameters by @returns_* decorators return_info = _fun.ovs_metadata.get('returns', None) if return_info is not None: # Extra parameters params = return_info['parameters'] fields = [] if 'contents' in params or 'sorting' in params: _cls = return_info['object_type'] fields = [prop.name for prop in _cls._properties] + \ ['{0}_guid'.format(rel.name) for rel in _cls._relations] + \ [dynamic.name for dynamic in _cls._dynamics] relation_info = RelationMapper.load_foreign_relations(_cls) if relation_info is not None: fields += [('{0}_guid' if rel_info['list'] is False else '{0}_guids').format(key) for key, rel_info in relation_info.iteritems()] fields = fields + ['-{0}'.format(field) for field in fields] for parameter in params: if parameter == 'contents': parameter_info.append({'name': 'contents', 'in': 'query', 'description': 'Specify the returned contents.', 'required': True, 'collectionFormat': 'csv', 'type': 'array', 'enum': ['_dynamics', '_relations', 'guid'] + fields, 'items': {'type': 'string'}}) elif parameter == 'paging': parameter_info.append({'name': 'page', 'in': 'query', 'description': 'Specifies the page to be returned.', 'required': False, 'type': 'integer'}) parameter_info.append({'name': 'page_size', 'in': 'query', 'description': 'Specifies the size of a page. Supported values: 10, 25, 50 and 100. Requires "page" to be set.', 'required': False, 'type': 'integer'}) elif parameter == 'sorting': parameter_info.append({'name': 'sort', 'in': 'query', 'description': 'Specifies the sorting of the list.', 'required': False, 'default': params[parameter], 'enum': ['guid', '-guid'] + fields, 'type': 'array', 'items': {'type': 'string'}}) return parameter_info def load_response(_fun): response_code = '200' response_schema = None return_info = _fun.ovs_metadata.get('returns', None) if return_info is not None: return_type, _return_code = return_info['returns'] if _return_code is not None: response_code = _return_code if return_type == 'object': _cls = return_info['object_type'] response_schema = {'$ref': '#/definitions/{0}'.format(_cls.__name__)} elif return_type == 'list': _cls = return_info['object_type'] class_schema = {'$ref': '#/definitions/{0}'.format(_cls.__name__)} fields = [prop.name for prop in _cls._properties] + \ ['{0}_guid'.format(rel.name) for rel in _cls._relations] + \ [dynamic.name for dynamic in _cls._dynamics] relation_info = RelationMapper.load_foreign_relations(_cls) if relation_info is not None: fields += [('{0}_guid' if rel_info['list'] is False else '{0}_guids').format(key) for key, rel_info in relation_info.iteritems()] fields = fields + ['-{0}'.format(field) for field in fields] response_schema = {'type': 'object', 'title': 'DataList', 'properties': {'_contents': {'type': 'array', 'description': 'Requested contents.', 'items': {'type': 'string'}, 'required': True, 'collectionFormat': 'csv', 'enum': ['_dynamics', '_relations', 'guid'] + fields}, '_paging': {'type': 'object', 'title': 'PagingMetadata', 'properties': {'total_items': {'type': 'integer', 'description': 'Total items available.'}, 'max_page': {'type': 'integer', 'description': 'Last page available.'}, 'end_number': {'type': 'integer', 'description': '1-based index of the last item in the current page.'}, 'current_page': {'type': 'integer', 'description': 'Current page number.'}, 'page_size': {'type': 'integer', 'description': 'Number of items in the current page.'}, 'start_number': {'type': 'integer', 'description': '1-based index of the first item in the current page'}}, 'required': ['total_items', 'max_page', 'end_number', 'current_page', 'page_size', 'start_number']}, '_sorting': {'type': 'array', 'description': 'Applied sorting', 'items': {'type': 'string'}, 'required': True, 'collectionFormat': 'csv', 'enum': ['-guid', 'guid'] + fields}, 'data': {'type': 'array', 'description': 'List of serialized {0}s.'.format(_cls.__name__), 'required': True, 'items': class_schema}}, 'required': ['_contents', '_paging', '_sorting', 'data']} else: docs = _fun.__doc__ doc_info = {} if docs is not None: for match in re.finditer(':(return|rtype): (.*)', docs, re.MULTILINE): entries = match.groups() doc_info[entries[0]] = entries[1] if return_type == 'task': task_return = '' if 'return' in doc_info: task_return = ' The task returns: {0}'.format(doc_info['return']) response_schema = {'type': 'string', 'description': 'A task identifier.{0}'.format(task_return)} elif return_type is None: response_schema = {'type': 'string'} if 'return' in doc_info: response_schema['description'] = doc_info['return'] if 'rtype' in doc_info: type_info = doc_info['rtype'] if type_info in ['int', 'long']: response_schema['type'] = 'integer' elif type_info in ['float']: response_schema['type'] = 'number' elif type_info in ['bool']: response_schema['type'] = 'boolean' elif type_info in ['str', 'basestring', 'unicode']: response_schema['type'] = 'string' elif type_info in ['dict']: response_schema['type'] = 'object' elif type_info in ['None']: response_schema = None response_code = '204' return response_code, response_schema paths = data['paths'] path = '/'.join([os.path.dirname(__file__), 'backend', 'views']) for filename in os.listdir(path): if os.path.isfile('/'.join([path, filename])) and filename.endswith('.py'): name = filename.replace('.py', '') module = imp.load_source(name, '/'.join([path, filename])) for member in inspect.getmembers(module): if inspect.isclass(member[1]) \ and member[1].__module__ == name \ and 'ViewSet' in [base.__name__ for base in member[1].__bases__]: cls = member[1] if hasattr(cls, 'skip_spec') and cls.skip_spec is True: continue base_calls = {'list': ['get', '/{0}/'], 'retrieve': ['get', '/{0}/{{guid}}/'], 'create': ['post', '/{0}/'], 'destroy': ['delete', '/{0}/{{guid}}/'], 'partial_update': ['patch', '/{0}/{{guid}}/']} for call, route_data in base_calls.iteritems(): if hasattr(cls, call): fun = getattr(cls, call) docstring = fun.__doc__.strip().split('\n')[0] parameters = load_parameters(fun) return_code, schema = load_response(fun) route = {route_data[0]: {'summary': docstring, 'operationId': '{0}.{1}'.format(member[1].prefix, call), 'responses': {return_code: {'description': docstring}, 'default': {'description': 'Error payload', 'schema': {'$ref': '#/definitions/APIError'}}}, 'parameters': parameters}} if schema is not None: route[route_data[0]]['responses'][return_code]['schema'] = schema current_path = route_data[1].format(member[1].prefix) if current_path not in paths: paths[current_path] = {} paths[current_path].update(route) funs = [fun[1] for fun in inspect.getmembers(cls, predicate=inspect.ismethod) if fun[0] not in base_calls.keys()] for fun in funs: if hasattr(fun, 'bind_to_methods'): routes = {} docstring = fun.__doc__.strip().split('\n')[0] parameters = load_parameters(fun) return_code, schema = load_response(fun) name = fun.__name__ for verb in fun.bind_to_methods: routes[verb] = {'summary': docstring, 'operationId': '{0}.{1}_{2}'.format(member[1].prefix, verb, name), 'responses': {return_code: {'description': docstring}, 'default': {'description': 'Error payload', 'schema': {'$ref': '#/definitions/APIError'}}}, 'parameters': parameters} if schema is not None: routes[verb]['responses'][return_code]['schema'] = schema paths['/{0}/{{guid}}/{1}/'.format(member[1].prefix, name)] = routes # DataObject / hybrids def build_property(prop): _docstring = prop.docstring or prop.name _docstring = _docstring.replace('None', 'null').replace('True', 'true').replace('False', 'false') info = {'description': _docstring} if prop.default is not None: info['default'] = prop.default if prop.property_type == int: info['type'] = 'integer' elif prop.property_type == float: info['type'] = 'number' elif prop.property_type == long: info['type'] = 'integer' elif prop.property_type == str: info['type'] = 'string' elif prop.property_type == bool: info['type'] = 'boolean' elif prop.property_type == list: info['type'] = 'array' elif prop.property_type == dict: info['type'] = 'object' elif prop.property_type == set: info['type'] = 'array' elif isinstance(prop.property_type, list): # enumerator info['type'] = 'string' info['enum'] = prop.property_type return info def build_relation(_cls, relation): itemtype = relation.foreign_type.__name__ if relation.foreign_type is not None else _cls.__name__ _docstring = '{1} instance identifier{3}. One-to-{0} relation with {1}.{2}.'.format( 'one' if relation.onetoone is True else 'many', itemtype, ('{0}_guid' if relation.onetoone is True else '{0}_guids').format(relation.foreign_key), '' if relation.mandatory is True else ', null if relation is not set' ) info = {'description': _docstring, 'type': 'string'} return '{0}_guid'.format(relation.name), info def build_dynamic(_cls, dynamic): _docstring = dynamic.name if hasattr(_cls, '_{0}'.format(dynamic.name)): docs = getattr(_cls, '_{0}'.format(dynamic.name)).__doc__ if docs is not None: _docstring = docs.strip().split('\n')[0] _docstring = _docstring.replace('None', 'null').replace('True', 'true').replace('False', 'false') _docstring = '{0} (dynamic property, cache timeout: {1}s)'.format(_docstring, dynamic.timeout) info = {'description': _docstring, 'readOnly': True} if dynamic.return_type == int: info['type'] = 'integer' elif dynamic.return_type == float: info['type'] = 'number' elif dynamic.return_type == long: info['type'] = 'integer' elif dynamic.return_type == str: info['type'] = 'string' elif dynamic.return_type == bool: info['type'] = 'boolean' elif dynamic.return_type == list: info['type'] = 'array' elif dynamic.return_type == dict: info['type'] = 'object' elif dynamic.return_type == set: info['type'] = 'array' elif isinstance(dynamic.return_type, list): # enumerator info['type'] = 'string' info['enum'] = dynamic.return_type return info def build_remote_relation(relation): key, relation_info = relation remote_cls = Descriptor().load(relation_info['class']).get_object() _docstring = '{1} instance identifier{3}. One-to-{0} relation with {1}.{2}.'.format( 'one' if relation_info['list'] is False else 'many', remote_cls.__name__, '{0}_guid'.format(relation_info['key']), '' if relation_info['list'] is False else 's' ) info = {'description': _docstring, 'readOnly': True} if relation_info['list'] is True: info['type'] = 'array' info['items'] = {'type': 'string'} _name = '{0}_guids'.format(key) else: info['type'] = 'string' _name = '{0}_guid'.format(key) return _name, info def get_properties(_cls): properties = {} properties.update({prop.name: build_property(prop) for prop in _cls._properties}) properties.update(dict(build_relation(_cls, relation) for relation in _cls._relations)) properties.update({dynamic.name: build_dynamic(_cls, dynamic) for dynamic in _cls._dynamics}) relation_info = RelationMapper.load_foreign_relations(_cls) if relation_info is not None: properties.update(dict(build_remote_relation(relation) for relation in relation_info.iteritems())) return properties def get_required_properties(_cls): required = [] for prop in _cls._properties: if prop.mandatory is True: required.append(prop.name) for relation in _cls._relations: if relation.mandatory is True: required.append('{0}_guid'.format(relation.name)) return required definitions = data['definitions'] definitions['DataObject'] = {'type': 'object', 'title': 'DataObject', 'description': 'Root object inherited by all hybrid objects. Shall not be used directly.', 'properties': {'guid': {'type': 'string', 'description': 'Identifier of the object.'}}, 'required': ['guid']} hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): cls = Descriptor().load(class_descriptor).get_object() definitions[cls.__name__] = {'description': cls.__doc__.strip().split('\n')[0], 'allOf': [{'$ref': '#/definitions/DataObject'}, {'type': 'object', 'properties': get_properties(cls), 'required': get_required_properties(cls)}]} return data
def migrate(previous_version): """ Migrates from a given version to the current version. It uses 'previous_version' to be smart wherever possible, but the code should be able to migrate any version towards the expected version. When this is not possible, the code can set a minimum version and raise when it is not met. :param previous_version: The previous version from which to start the migration :type previous_version: float """ working_version = previous_version if working_version == 0: # Initial version: # * Add any basic configuration or model entries # Add backends for backend_type_info in [('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in [ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # From here on, all actual migration should happen to get to the expected state for THIS RELEASE elif working_version < ALBAMigrator.THIS_VERSION: # Migrate unique constraints import hashlib from ovs.dal.helpers import HybridRunner, Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): cls = Descriptor().load(class_descriptor).get_object() classname = cls.__name__.lower() unique_key = 'ovs_unique_{0}_{{0}}_'.format(classname) uniques = [] # noinspection PyProtectedMember for prop in cls._properties: if prop.unique is True and len([k for k in client.prefix(unique_key.format(prop.name))]) == 0: uniques.append(prop.name) if len(uniques) > 0: prefix = 'ovs_data_{0}_'.format(classname) for key in client.prefix(prefix): data = client.get(key) for property_name in uniques: ukey = '{0}{1}'.format(unique_key.format(property_name), hashlib.sha1(str(data[property_name])).hexdigest()) client.set(ukey, key) # Changes on AlbaNodes & AlbaDisks from ovs.dal.lists.albanodelist import AlbaNodeList storagerouter_guids = [] for alba_node in AlbaNodeList.get_albanodes(): # StorageRouter - AlbaNode 1-to-many relation changes to 1-to-1 if alba_node.storagerouter_guid is not None: if alba_node.storagerouter_guid in storagerouter_guids: alba_node.storagerouter = None alba_node.save() else: storagerouter_guids.append(alba_node.storagerouter_guid) # Complete rework of the way we detect devices to assign roles or use as ASD # Allow loop-, raid-, nvme-, ??-devices and logical volumes as ASD # More info: https://github.com/openvstorage/framework/issues/792 for alba_disk in alba_node.disks: if alba_disk.aliases is not None: continue if 'name' in alba_disk._data: alba_disk.aliases = ['/dev/disk/by-id/{0}'.format(alba_disk._data['name'])] alba_disk.save() return ALBAMigrator.THIS_VERSION
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at version 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 1 introduced: # - The datastore is still empty, add defaults if working_version < 1: from ovs.dal.hybrids.user import User from ovs.dal.hybrids.group import Group from ovs.dal.hybrids.role import Role from ovs.dal.hybrids.client import Client from ovs.dal.hybrids.failuredomain import FailureDomain from ovs.dal.hybrids.j_rolegroup import RoleGroup from ovs.dal.hybrids.j_roleclient import RoleClient from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.branding import Branding from ovs.dal.lists.backendtypelist import BackendTypeList # Create groups admin_group = Group() admin_group.name = 'administrators' admin_group.description = 'Administrators' admin_group.save() viewers_group = Group() viewers_group.name = 'viewers' viewers_group.description = 'Viewers' viewers_group.save() # Create users admin = User() admin.username = '******' admin.password = hashlib.sha256('admin').hexdigest() admin.is_active = True admin.group = admin_group admin.save() # Create internal OAuth 2 clients admin_pw_client = Client() admin_pw_client.ovs_type = 'INTERNAL' admin_pw_client.grant_type = 'PASSWORD' admin_pw_client.user = admin admin_pw_client.save() admin_cc_client = Client() admin_cc_client.ovs_type = 'INTERNAL' admin_cc_client.grant_type = 'CLIENT_CREDENTIALS' admin_cc_client.client_secret = ''.join( random.choice(string.ascii_letters + string.digits + '|_=+*#@!/-[]{}<>.?,\'";:~') for _ in range(128)) admin_cc_client.user = admin admin_cc_client.save() # Create roles read_role = Role() read_role.code = 'read' read_role.name = 'Read' read_role.description = 'Can read objects' read_role.save() write_role = Role() write_role.code = 'write' write_role.name = 'Write' write_role.description = 'Can write objects' write_role.save() manage_role = Role() manage_role.code = 'manage' manage_role.name = 'Manage' manage_role.description = 'Can manage the system' manage_role.save() # Attach groups to roles mapping = [(admin_group, [read_role, write_role, manage_role]), (viewers_group, [read_role])] for setting in mapping: for role in setting[1]: rolegroup = RoleGroup() rolegroup.group = setting[0] rolegroup.role = role rolegroup.save() for user in setting[0].users: for role in setting[1]: for client in user.clients: roleclient = RoleClient() roleclient.client = client roleclient.role = role roleclient.save() # Add backends for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'), ('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in [ ServiceType.SERVICE_TYPES.MD_SERVER, ServiceType.SERVICE_TYPES.ALBA_PROXY, ServiceType.SERVICE_TYPES.ARAKOON ]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # Branding branding = Branding() branding.name = 'Default' branding.description = 'Default bootstrap theme' branding.css = 'bootstrap-default.min.css' branding.productname = 'Open vStorage' branding.is_default = True branding.save() slate = Branding() slate.name = 'Slate' slate.description = 'Dark bootstrap theme' slate.css = 'bootstrap-slate.min.css' slate.productname = 'Open vStorage' slate.is_default = False slate.save() # Failure Domain failure_domain = FailureDomain() failure_domain.name = 'Default' failure_domain.save() # We're now at version 1 working_version = 1 # Version 2 introduced: # - new Descriptor format if working_version < 2: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data') for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[ entry] and 'hybrids' in data[entry]['source']: filename = data[entry]['source'] if not filename.startswith('/'): filename = '/opt/OpenvStorage/ovs/dal/{0}'.format( filename) module = imp.load_source(data[entry]['name'], filename) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) # We're now at version 2 working_version = 2 # Version 3 introduced: # - new Descriptor format if working_version < 3: import imp from ovs.dal.helpers import Descriptor from ovs.extensions.storage.persistentfactory import PersistentFactory client = PersistentFactory.get_client() keys = client.prefix('ovs_data') for key in keys: data = client.get(key) modified = False for entry in data.keys(): if isinstance(data[entry], dict) and 'source' in data[entry]: module = imp.load_source(data[entry]['name'], data[entry]['source']) cls = getattr(module, data[entry]['type']) new_data = Descriptor(cls, cached=False).descriptor if 'guid' in data[entry]: new_data['guid'] = data[entry]['guid'] data[entry] = new_data modified = True if modified is True: data['_version'] += 1 client.set(key, data) working_version = 3 # Version 4 introduced: # - Flexible SSD layout if working_version < 4: import os from ovs.dal.hybrids.diskpartition import DiskPartition from ovs.dal.hybrids.j_storagedriverpartition import StorageDriverPartition from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.lists.servicetypelist import ServiceTypeList from ovs.dal.lists.storagedriverlist import StorageDriverList from ovs.extensions.generic.remote import remote from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration for service in ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.MD_SERVER).services: mds_service = service.mds_service storagedriver = None for current_storagedriver in service.storagerouter.storagedrivers: if current_storagedriver.vpool_guid == mds_service.vpool_guid: storagedriver = current_storagedriver break tasks = {} if storagedriver._data.get('mountpoint_md'): tasks['{0}/mds_{1}_{2}'.format( storagedriver._data.get('mountpoint_md'), storagedriver.vpool.name, mds_service.number)] = ( DiskPartition.ROLES.DB, StorageDriverPartition.SUBROLE.MDS) if storagedriver._data.get('mountpoint_temp'): tasks['{0}/mds_{1}_{2}'.format( storagedriver._data.get('mountpoint_temp'), storagedriver.vpool.name, mds_service.number)] = ( DiskPartition.ROLES.SCRUB, StorageDriverPartition.SUBROLE.MDS) for disk in service.storagerouter.disks: for partition in disk.partitions: for directory, (role, subrole) in tasks.iteritems(): with remote(storagedriver.storagerouter.ip, [os], username='******') as rem: stat_dir = directory while not rem.os.path.exists( stat_dir) and stat_dir != '/': stat_dir = stat_dir.rsplit('/', 1)[0] if not stat_dir: stat_dir = '/' inode = rem.os.stat(stat_dir).st_dev if partition.inode == inode: if role not in partition.roles: partition.roles.append(role) partition.save() number = 0 migrated = False for sd_partition in storagedriver.partitions: if sd_partition.role == role and sd_partition.sub_role == subrole: if sd_partition.mds_service == mds_service: migrated = True break if sd_partition.partition_guid == partition.guid: number = max( sd_partition.number, number) if migrated is False: sd_partition = StorageDriverPartition() sd_partition.role = role sd_partition.sub_role = subrole sd_partition.partition = partition sd_partition.storagedriver = storagedriver sd_partition.mds_service = mds_service sd_partition.size = None sd_partition.number = number + 1 sd_partition.save() client = SSHClient( storagedriver.storagerouter, username='******') path = sd_partition.path.rsplit('/', 1)[0] if path: client.dir_create(path) client.dir_chown(path, 'ovs', 'ovs') client.dir_create(directory) client.dir_chown(directory, 'ovs', 'ovs') client.symlink( {sd_partition.path: directory}) for storagedriver in StorageDriverList.get_storagedrivers(): migrated_objects = {} for disk in storagedriver.storagerouter.disks: for partition in disk.partitions: # Process all mountpoints that are unique and don't have a specified size for key, (role, sr_info) in { 'mountpoint_md': (DiskPartition.ROLES.DB, { 'metadata_{0}': StorageDriverPartition.SUBROLE.MD, 'tlogs_{0}': StorageDriverPartition.SUBROLE.TLOG }), 'mountpoint_fragmentcache': (DiskPartition.ROLES.WRITE, { 'fcache_{0}': StorageDriverPartition.SUBROLE.FCACHE }), 'mountpoint_foc': (DiskPartition.ROLES.WRITE, { 'fd_{0}': StorageDriverPartition.SUBROLE.FD, 'dtl_{0}': StorageDriverPartition.SUBROLE.DTL }), 'mountpoint_dtl': (DiskPartition.ROLES.WRITE, { 'fd_{0}': StorageDriverPartition.SUBROLE.FD, 'dtl_{0}': StorageDriverPartition.SUBROLE.DTL }), 'mountpoint_readcaches': (DiskPartition.ROLES.READ, { '': None }), 'mountpoint_writecaches': (DiskPartition.ROLES.WRITE, { 'sco_{0}': StorageDriverPartition.SUBROLE.SCO }) }.iteritems(): if key in storagedriver._data: is_list = isinstance(storagedriver._data[key], list) entries = storagedriver._data[ key][:] if is_list is True else [ storagedriver._data[key] ] for entry in entries: if not entry: if is_list: storagedriver._data[key].remove( entry) if len(storagedriver._data[key] ) == 0: del storagedriver._data[key] else: del storagedriver._data[key] else: with remote( storagedriver.storagerouter.ip, [os], username='******') as rem: inode = rem.os.stat(entry).st_dev if partition.inode == inode: if role not in partition.roles: partition.roles.append(role) partition.save() for folder, subrole in sr_info.iteritems( ): number = 0 migrated = False for sd_partition in storagedriver.partitions: if sd_partition.role == role and sd_partition.sub_role == subrole: if sd_partition.partition_guid == partition.guid: number = max( sd_partition. number, number) if migrated is False: sd_partition = StorageDriverPartition( ) sd_partition.role = role sd_partition.sub_role = subrole sd_partition.partition = partition sd_partition.storagedriver = storagedriver sd_partition.size = None sd_partition.number = number + 1 sd_partition.save() if folder: source = '{0}/{1}'.format( entry, folder.format( storagedriver. vpool.name)) else: source = entry client = SSHClient( storagedriver. storagerouter, username='******') path = sd_partition.path.rsplit( '/', 1)[0] if path: client.dir_create(path) client.dir_chown( path, 'ovs', 'ovs') client.symlink({ sd_partition.path: source }) migrated_objects[ source] = sd_partition if is_list: storagedriver._data[ key].remove(entry) if len(storagedriver._data[key] ) == 0: del storagedriver._data[ key] else: del storagedriver._data[key] storagedriver.save() if 'mountpoint_bfs' in storagedriver._data: storagedriver.mountpoint_dfs = storagedriver._data[ 'mountpoint_bfs'] if not storagedriver.mountpoint_dfs: storagedriver.mountpoint_dfs = None del storagedriver._data['mountpoint_bfs'] storagedriver.save() if 'mountpoint_temp' in storagedriver._data: del storagedriver._data['mountpoint_temp'] storagedriver.save() if migrated_objects: print 'Loading sizes' config = StorageDriverConfiguration( 'storagedriver', storagedriver.vpool_guid, storagedriver.storagedriver_id) config.load() for readcache in config.configuration.get( 'content_addressed_cache', {}).get('clustercache_mount_points', []): path = readcache.get('path', '').rsplit('/', 1)[0] size = int(readcache['size'].strip( 'KiB')) * 1024 if 'size' in readcache else None if path in migrated_objects: migrated_objects[path].size = long(size) migrated_objects[path].save() for writecache in config.configuration.get( 'scocache', {}).get('scocache_mount_points', []): path = writecache.get('path', '') size = int(writecache['size'].strip( 'KiB')) * 1024 if 'size' in writecache else None if path in migrated_objects: migrated_objects[path].size = long(size) migrated_objects[path].save() working_version = 4 # Version 5 introduced: # - Failure Domains if working_version < 5: import os from ovs.dal.hybrids.failuredomain import FailureDomain from ovs.dal.lists.failuredomainlist import FailureDomainList from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.extensions.generic.remote import remote from ovs.extensions.generic.sshclient import SSHClient failure_domains = FailureDomainList.get_failure_domains() if len(failure_domains) > 0: failure_domain = failure_domains[0] else: failure_domain = FailureDomain() failure_domain.name = 'Default' failure_domain.save() for storagerouter in StorageRouterList.get_storagerouters(): change = False if storagerouter.primary_failure_domain is None: storagerouter.primary_failure_domain = failure_domain change = True if storagerouter.rdma_capable is None: client = SSHClient(storagerouter, username='******') rdma_capable = False with remote(client.ip, [os], username='******') as rem: for root, dirs, files in rem.os.walk( '/sys/class/infiniband'): for directory in dirs: ports_dir = '/'.join( [root, directory, 'ports']) if not rem.os.path.exists(ports_dir): continue for sub_root, sub_dirs, _ in rem.os.walk( ports_dir): if sub_root != ports_dir: continue for sub_directory in sub_dirs: state_file = '/'.join( [sub_root, sub_directory, 'state']) if rem.os.path.exists(state_file): if 'ACTIVE' in client.run( 'cat {0}'.format( state_file)): rdma_capable = True storagerouter.rdma_capable = rdma_capable change = True if change is True: storagerouter.save() working_version = 5 # Version 6 introduced: # - Distributed scrubbing if working_version < 6: from ovs.dal.hybrids.diskpartition import DiskPartition from ovs.dal.lists.storagedriverlist import StorageDriverList from ovs.extensions.generic.sshclient import SSHClient for storage_driver in StorageDriverList.get_storagedrivers(): root_client = SSHClient(storage_driver.storagerouter, username='******') for partition in storage_driver.partitions: if partition.role == DiskPartition.ROLES.SCRUB: old_path = partition.path partition.sub_role = None partition.save() partition.invalidate_dynamics(['folder', 'path']) if root_client.dir_exists(partition.path): continue # New directory already exists if '_mds_' in old_path: if root_client.dir_exists(old_path): root_client.symlink({partition.path: old_path}) if not root_client.dir_exists(partition.path): root_client.dir_create(partition.path) root_client.dir_chmod(partition.path, 0777) working_version = 6 # Version 7 introduced: # - vPool status if working_version < 7: from ovs.dal.hybrids import vpool reload(vpool) from ovs.dal.hybrids.vpool import VPool from ovs.dal.lists.vpoollist import VPoolList for _vpool in VPoolList.get_vpools(): vpool = VPool(_vpool.guid) if hasattr(vpool, 'status') and vpool.status is None: vpool.status = VPool.STATUSES.RUNNING vpool.save() working_version = 7 # Version 10 introduced: # - Reverse indexes are stored in persistent store # - Store more non-changing metadata on disk iso using a dynamic property if working_version < 10: from ovs.dal.helpers import HybridRunner, Descriptor from ovs.dal.datalist import DataList from ovs.extensions.storage.persistentfactory import PersistentFactory from ovs.extensions.storage.volatilefactory import VolatileFactory persistent = PersistentFactory.get_client() for prefix in ['ovs_listcache', 'ovs_reverseindex']: for key in persistent.prefix(prefix): persistent.delete(key) for key in persistent.prefix('ovs_data_'): persistent.set(key, persistent.get(key)) base_reverse_key = 'ovs_reverseindex_{0}_{1}|{2}|{3}' hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): cls = Descriptor().load(class_descriptor).get_object() all_objects = DataList(cls, { 'type': DataList.where_operator.AND, 'items': [] }) for item in all_objects: guid = item.guid for relation in item._relations: if relation.foreign_type is None: rcls = cls rclsname = rcls.__name__.lower() else: rcls = relation.foreign_type rclsname = rcls.__name__.lower() key = relation.name rguid = item._data[key]['guid'] if rguid is not None: reverse_key = base_reverse_key.format( rclsname, rguid, relation.foreign_key, guid) persistent.set(reverse_key, 0) volatile = VolatileFactory.get_client() try: volatile._client.flush_all() except: pass from ovs.dal.lists.vdisklist import VDiskList for vdisk in VDiskList.get_vdisks(): try: vdisk.metadata = { 'lba_size': vdisk.info['lba_size'], 'cluster_multiplier': vdisk.info['cluster_multiplier'] } vdisk.save() except: pass working_version = 10 # Version 11 introduced: # - ALBA accelerated ALBA, meaning different vpool.metadata information if working_version < 11: from ovs.dal.lists.vpoollist import VPoolList for vpool in VPoolList.get_vpools(): vpool.metadata = {'backend': vpool.metadata} if 'metadata' in vpool.metadata['backend']: vpool.metadata['backend'][ 'arakoon_config'] = vpool.metadata['backend'].pop( 'metadata') if 'backend_info' in vpool.metadata['backend']: vpool.metadata['backend']['backend_info'][ 'fragment_cache_on_read'] = True vpool.metadata['backend']['backend_info'][ 'fragment_cache_on_write'] = False vpool.save() working_version = 11 return working_version
def get(self, request, *args, **kwargs): """ Fetches metadata """ _ = args, kwargs data = {'authenticated': False, 'authentication_state': None, 'authentication_metadata': {}, 'username': None, 'userguid': None, 'roles': [], 'identification': {}, 'storagerouter_ips': [sr.ip for sr in StorageRouterList.get_storagerouters()], 'versions': list(settings.VERSION), 'plugins': {}, 'registration': {'registered': False, 'remaining': None}} try: # Gather plugin metadata plugins = {} # - Backends. BackendType plugins must set the has_plugin flag on True for backend_type in BackendTypeList.get_backend_types(): if backend_type.has_plugin is True: if backend_type.code not in plugins: plugins[backend_type.code] = [] plugins[backend_type.code] += ['backend', 'gui'] # - Generic plugins, as added to the configuration file(s) generic_plugins = Configuration.get('ovs.plugins.generic') for plugin_name in generic_plugins: if plugin_name not in plugins: plugins[plugin_name] = [] plugins[plugin_name] += ['gui'] data['plugins'] = plugins # Fill identification data['identification'] = {'cluster_id': Configuration.get('ovs.support.cid')} # Registration data registered = Configuration.get('ovs.core.registered') data['registration']['registered'] = registered if registered is False: cluster_install_time = None for storagerouter in StorageRouterList.get_storagerouters(): client = SSHClient(storagerouter) install_time = client.config_read('ovs.core.install_time') if cluster_install_time is None or (install_time is not None and install_time < cluster_install_time): cluster_install_time = install_time if cluster_install_time is not None: timeout_days = 30 * 24 * 60 * 60 data['registration']['remaining'] = (timeout_days - time.time() + cluster_install_time) / 24 / 60 / 60 # Get authentication metadata authentication_metadata = {'ip': System.get_my_storagerouter().ip} for key in ['mode', 'authorize_uri', 'client_id', 'scope']: if Configuration.exists('ovs.webapps.oauth2.{0}'.format(key)): authentication_metadata[key] = Configuration.get('ovs.webapps.oauth2.{0}'.format(key)) data['authentication_metadata'] = authentication_metadata # Gather authorization metadata if 'HTTP_AUTHORIZATION' not in request.META: return HttpResponse, dict(data.items() + {'authentication_state': 'unauthenticated'}.items()) authorization_type, access_token = request.META['HTTP_AUTHORIZATION'].split(' ') if authorization_type != 'Bearer': return HttpResponse, dict(data.items() + {'authentication_state': 'invalid_authorization_type'}.items()) tokens = BearerTokenList.get_by_access_token(access_token) if len(tokens) != 1: return HttpResponse, dict(data.items() + {'authentication_state': 'invalid_token'}.items()) token = tokens[0] if token.expiration < time.time(): for junction in token.roles.itersafe(): junction.delete() token.delete() return HttpResponse, dict(data.items() + {'authentication_state': 'token_expired'}.items()) # Gather user metadata user = token.client.user if not user.is_active: return HttpResponse, dict(data.items() + {'authentication_state': 'inactive_user'}.items()) roles = [j.role.code for j in token.roles] return HttpResponse, dict(data.items() + {'authenticated': True, 'authentication_state': 'authenticated', 'username': user.username, 'userguid': user.guid, 'roles': roles, 'plugins': plugins}.items()) except Exception as ex: logger.exception('Unexpected exception: {0}'.format(ex)) return HttpResponse, dict(data.items() + {'authentication_state': 'unexpected_exception'}.items())
def migrate(previous_version): """ Migrates from a given version to the current version. It uses 'previous_version' to be smart wherever possible, but the code should be able to migrate any version towards the expected version. When this is not possible, the code can set a minimum version and raise when it is not met. :param previous_version: The previous version from which to start the migration :type previous_version: float """ working_version = previous_version if working_version == 0: from ovs.dal.hybrids.servicetype import ServiceType # Initial version: # * Add any basic configuration or model entries # Add backends for backend_type_info in [('ALBA', 'alba')]: code = backend_type_info[1] backend_type = BackendTypeList.get_backend_type_by_code(code) if backend_type is None: backend_type = BackendType() backend_type.name = backend_type_info[0] backend_type.code = code backend_type.save() # Add service types for service_type_info in [ ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR, ServiceType.SERVICE_TYPES.ALBA_S3_TRANSACTION ]: service_type = ServiceType() service_type.name = service_type_info service_type.save() # From here on, all actual migration should happen to get to the expected state for THIS RELEASE elif working_version < DALMigrator.THIS_VERSION: import hashlib from ovs.dal.exceptions import ObjectNotFoundException from ovs.dal.helpers import HybridRunner, Descriptor from ovs.dal.hybrids.albaabmcluster import ABMCluster from ovs.dal.hybrids.albaosd import AlbaOSD from ovs.dal.hybrids.albansmcluster import NSMCluster from ovs.dal.hybrids.j_abmservice import ABMService from ovs.dal.hybrids.j_nsmservice import NSMService from ovs.dal.hybrids.service import Service from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.lists.albabackendlist import AlbaBackendList from ovs.dal.lists.albanodelist import AlbaNodeList from ovs.dal.lists.servicetypelist import ServiceTypeList from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.extensions.db.arakooninstaller import ArakoonClusterConfig, ArakoonInstaller from ovs.extensions.generic.configuration import Configuration, NotFoundException from ovs_extensions.generic.toolbox import ExtensionsToolbox from ovs.extensions.plugins.albacli import AlbaCLI from ovs.extensions.storage.persistentfactory import PersistentFactory # Migrate unique constraints & indexes client = PersistentFactory.get_client() hybrid_structure = HybridRunner.get_hybrids() for class_descriptor in hybrid_structure.values(): cls = Descriptor().load(class_descriptor).get_object() classname = cls.__name__.lower() unique_key = 'ovs_unique_{0}_{{0}}_'.format(classname) index_prefix = 'ovs_index_{0}|{{0}}|'.format(classname) index_key = 'ovs_index_{0}|{{0}}|{{1}}'.format(classname) uniques = [] indexes = [] # noinspection PyProtectedMember for prop in cls._properties: if prop.unique is True and len([ k for k in client.prefix( unique_key.format(prop.name)) ]) == 0: uniques.append(prop.name) if prop.indexed is True and len([ k for k in client.prefix( index_prefix.format(prop.name)) ]) == 0: indexes.append(prop.name) if len(uniques) > 0 or len(indexes) > 0: prefix = 'ovs_data_{0}_'.format(classname) for key, data in client.prefix_entries(prefix): for property_name in uniques: ukey = '{0}{1}'.format( unique_key.format(property_name), hashlib.sha1(str( data[property_name])).hexdigest()) client.set(ukey, key) for property_name in indexes: if property_name not in data: continue # This is the case when there's a new indexed property added. ikey = index_key.format( property_name, hashlib.sha1(str( data[property_name])).hexdigest()) index = list( client.get_multi([ikey], must_exist=False))[0] transaction = client.begin_transaction() if index is None: client.assert_value(ikey, None, transaction=transaction) client.set(ikey, [key], transaction=transaction) elif key not in index: client.assert_value(ikey, index[:], transaction=transaction) client.set(ikey, index + [key], transaction=transaction) client.apply_transaction(transaction) ############################################# # Introduction of ABMCluster and NSMCluster # ############################################# # Verify presence of unchanged ALBA Backends alba_backends = AlbaBackendList.get_albabackends() changes_required = False for alba_backend in alba_backends: if alba_backend.abm_cluster is None or len( alba_backend.nsm_clusters) == 0: changes_required = True break if changes_required: # Retrieve ABM and NSM clusters abm_cluster_info = [] nsm_cluster_info = [] for cluster_name in Configuration.list('/ovs/arakoon'): try: metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=cluster_name) if metadata[ 'cluster_type'] == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: abm_cluster_info.append(metadata) elif metadata[ 'cluster_type'] == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: nsm_cluster_info.append(metadata) except NotFoundException: continue # Retrieve NSM Arakoon cluster information cluster_arakoon_map = {} for cluster_info in abm_cluster_info + nsm_cluster_info: cluster_name = cluster_info['cluster_name'] arakoon_config = ArakoonClusterConfig( cluster_id=cluster_name) cluster_arakoon_map[ cluster_name] = arakoon_config.export_dict() storagerouter_map = dict( (storagerouter.machine_id, storagerouter) for storagerouter in StorageRouterList.get_storagerouters()) alba_backend_id_map = dict((alba_backend.alba_id, alba_backend) for alba_backend in alba_backends) for cluster_info in abm_cluster_info: internal = cluster_info['internal'] cluster_name = cluster_info['cluster_name'] config_location = Configuration.get_configuration_path( key=ArakoonClusterConfig.CONFIG_KEY.format( cluster_name)) try: alba_id = AlbaCLI.run(command='get-alba-id', config=config_location, named_params={'attempts': 3})['id'] nsm_hosts = AlbaCLI.run(command='list-nsm-hosts', config=config_location, named_params={'attempts': 3}) except RuntimeError: continue alba_backend = alba_backend_id_map.get(alba_id) if alba_backend is None: # ALBA Backend with ID not found in model continue if alba_backend.abm_cluster is not None and len( alba_backend.nsm_clusters ) > 0: # Clusters already exist continue # Create ABM Cluster if alba_backend.abm_cluster is None: abm_cluster = ABMCluster() abm_cluster.name = cluster_name abm_cluster.alba_backend = alba_backend abm_cluster.config_location = ArakoonClusterConfig.CONFIG_KEY.format( cluster_name) abm_cluster.save() else: abm_cluster = alba_backend.abm_cluster # Create ABM Services abm_arakoon_config = cluster_arakoon_map[cluster_name] abm_arakoon_config.pop('global') arakoon_nodes = abm_arakoon_config.keys() if internal is False: services_to_create = 1 else: if set(arakoon_nodes).difference( set(storagerouter_map.keys())): continue services_to_create = len(arakoon_nodes) for index in range(services_to_create): service = Service() service.name = 'arakoon-{0}-abm'.format( alba_backend.name) service.type = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.ALBA_MGR) if internal is True: arakoon_node_config = abm_arakoon_config[ arakoon_nodes[index]] service.ports = [ arakoon_node_config['client_port'], arakoon_node_config['messaging_port'] ] service.storagerouter = storagerouter_map[ arakoon_nodes[index]] else: service.ports = [] service.storagerouter = None service.save() abm_service = ABMService() abm_service.service = service abm_service.abm_cluster = abm_cluster abm_service.save() # Create NSM Clusters for cluster_index, nsm_host in enumerate( sorted(nsm_hosts, key=lambda host: ExtensionsToolbox. advanced_sort(host['cluster_id'], '_'))): nsm_cluster_name = nsm_host['cluster_id'] nsm_arakoon_config = cluster_arakoon_map.get( nsm_cluster_name) if nsm_arakoon_config is None: continue number = cluster_index if internal is False else int( nsm_cluster_name.split('_')[-1]) nsm_cluster = NSMCluster() nsm_cluster.name = nsm_cluster_name nsm_cluster.number = number nsm_cluster.alba_backend = alba_backend nsm_cluster.config_location = ArakoonClusterConfig.CONFIG_KEY.format( nsm_cluster_name) nsm_cluster.save() # Create NSM Services nsm_arakoon_config.pop('global') arakoon_nodes = nsm_arakoon_config.keys() if internal is False: services_to_create = 1 else: if set(arakoon_nodes).difference( set(storagerouter_map.keys())): continue services_to_create = len(arakoon_nodes) for service_index in range(services_to_create): service = Service() service.name = 'arakoon-{0}-nsm_{1}'.format( alba_backend.name, number) service.type = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.NS_MGR) if internal is True: arakoon_node_config = nsm_arakoon_config[ arakoon_nodes[service_index]] service.ports = [ arakoon_node_config['client_port'], arakoon_node_config['messaging_port'] ] service.storagerouter = storagerouter_map[ arakoon_nodes[service_index]] else: service.ports = [] service.storagerouter = None service.save() nsm_service = NSMService() nsm_service.service = service nsm_service.nsm_cluster = nsm_cluster nsm_service.save() # Clean up all junction services no longer linked to an ALBA Backend all_nsm_services = [ service.nsm_service for service in ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.NS_MGR).services if service.nsm_service.nsm_cluster is None ] all_abm_services = [ service.abm_service for service in ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.ALBA_MGR).services if service.abm_service.abm_cluster is None ] for abm_service in all_abm_services: abm_service.delete() abm_service.service.delete() for nsm_service in all_nsm_services: nsm_service.delete() nsm_service.service.delete() ################################ # Introduction of Active Drive # ################################ # Update slot_id and Alba Node relation for all OSDs client = PersistentFactory.get_client() disk_osd_map = {} for key, data in client.prefix_entries('ovs_data_albaosd_'): alba_disk_guid = data.get('alba_disk', {}).get('guid') if alba_disk_guid is not None: if alba_disk_guid not in disk_osd_map: disk_osd_map[alba_disk_guid] = [] disk_osd_map[alba_disk_guid].append( key.replace('ovs_data_albaosd_', '')) try: value = client.get(key) value.pop('alba_disk', None) client.set(key=key, value=value) except Exception: pass # We don't care if we would have any leftover AlbaDisk information in _data, but its cleaner not to alba_guid_node_map = dict( (an.guid, an) for an in AlbaNodeList.get_albanodes()) for key, data in client.prefix_entries('ovs_data_albadisk_'): alba_disk_guid = key.replace('ovs_data_albadisk_', '') alba_node_guid = data.get('alba_node', {}).get('guid') if alba_disk_guid in disk_osd_map and alba_node_guid in alba_guid_node_map and len( data.get('aliases', [])) > 0: slot_id = data['aliases'][0].split('/')[-1] for osd_guid in disk_osd_map[alba_disk_guid]: try: osd = AlbaOSD(osd_guid) except ObjectNotFoundException: continue osd.slot_id = slot_id osd.alba_node = alba_guid_node_map[alba_node_guid] osd.save() client.delete(key=key, must_exist=False) # Remove unique constraints for AlbaNode IP for key in client.prefix('ovs_unique_albanode_ip_'): client.delete(key=key, must_exist=False) # Remove relation for all Alba Disks for key in client.prefix('ovs_reverseindex_albadisk_'): client.delete(key=key, must_exist=False) # Remove the relation between AlbaNode and AlbaDisk for key in client.prefix('ovs_reverseindex_albanode_'): if '|disks|' in key: client.delete(key=key, must_exist=False) return DALMigrator.THIS_VERSION
def list(self): """ Overview of all backend types """ return BackendTypeList.get_backend_types()