def create_virtblock_volume(hostvol_mnt, volname, size): """Create virtual block volume""" volhash = get_volname_hash(volname) volpath = get_volume_path(PV_TYPE_VIRTBLOCK, volhash, volname) volpath_full = os.path.join(hostvol_mnt, volpath) logging.debug(logf("Volume hash", volhash=volhash)) # Create a file with required size makedirs(os.path.dirname(volpath_full)) logging.debug( logf("Created virtblock directory", path=os.path.dirname(volpath))) volpath_fd = os.open(volpath_full, os.O_CREAT | os.O_RDWR) os.close(volpath_fd) os.truncate(volpath_full, size) logging.debug( logf("Truncated file to required size", path=volpath, size=size)) # TODO: Multiple FS support based on volume_capability mount option execute(MKFS_XFS_CMD, volpath_full) logging.debug( logf("Created Filesystem", path=volpath, command=MKFS_XFS_CMD)) save_pv_metadata(hostvol_mnt, volpath, size) return Volume( volname=volname, voltype=PV_TYPE_VIRTBLOCK, volhash=volhash, hostvol=os.path.basename(hostvol_mnt), size=size, volpath=volpath, )
def create_subdir_volume(hostvol_mnt, volname, size): """Create sub directory Volume""" volhash = get_volname_hash(volname) volpath = get_volume_path(PV_TYPE_SUBVOL, volhash, volname) logging.debug(logf("Volume hash", volhash=volhash)) # Check for mount availability before creating subdir volume retry_errors(os.statvfs, [hostvol_mnt], [ENOTCONN]) # Create a subdir makedirs(os.path.join(hostvol_mnt, volpath)) logging.debug(logf("Created PV directory", pvdir=volpath)) # Write info file so that Brick's quotad sidecar # container picks it up. save_pv_metadata(hostvol_mnt, volpath, size) # Wait for quota set # TODO: Handle Timeout pvsize_buffer = size * 0.05 # 5% pvsize_min = (size - pvsize_buffer) pvsize_max = (size + pvsize_buffer) logging.debug( logf( "Watching df of pv directory", pvdir=volpath, pvsize_buffer=pvsize_buffer, )) count = 0 while True: count += 1 pvstat = retry_errors(os.statvfs, [os.path.join(hostvol_mnt, volpath)], [ENOTCONN]) volsize = pvstat.f_blocks * pvstat.f_bsize if pvsize_min < volsize < pvsize_max: logging.debug( logf("Matching df output, Quota set successful", volsize=volsize, num_tries=count)) break if count >= 6: logging.warning( logf("Waited for some time, Quota set failed, continuing.", volsize=volsize, num_tries=count)) break time.sleep(1) return Volume( volname=volname, voltype=PV_TYPE_SUBVOL, volhash=volhash, hostvol=os.path.basename(hostvol_mnt), size=size, volpath=volpath, )
def mount_glusterfs_with_host(volname, target_path, host, options=None): """Mount Glusterfs Volume""" # Ignore if already mounted if is_gluster_mount_proc_running(volname, target_path): logging.debug(logf( "Already mounted", mount=target_path )) return if not os.path.exists(target_path): makedirs(target_path) # FIXME: make this better later (an issue for external contribution) # opt_array = None # if options: # opt_array = [] # for opt in options.split(","): # if not opt or opt == "": # break # for k,v in opt.split("="): # if k == "log-level": # opt_array.append("--log-level") # opt_array.append(v) # # TODO: handle more options, and document them # Fix the log, so we can check it out later # log_file = "/var/log/gluster/%s.log" % target_path.replace("/", "-") log_file = "/var/log/gluster/gluster.log" cmd = [ GLUSTERFS_CMD, "--process-name", "fuse", "-l", "%s" % log_file, "--volfile-id", volname, "-s", host, target_path ] # if opt_array: # cmd.extend(opt_array) # # # add mount point after all options # cmd.append(target_path) logging.debug(logf( "glusterfs command", cmd=cmd )) try: execute(*cmd) except CommandException as err: logging.info(logf( "mount command failed", cmd=cmd, error=err, )) return
def mount_glusterfs(volume, target_path): """Mount Glusterfs Volume""" if not os.path.exists(target_path): makedirs(target_path) # Ignore if already mounted if os.path.ismount(target_path): logging.debug(logf("Already mounted", mount=target_path)) return # This is just to prevent multiple requests getting here in parallel target_path_lock = "%s.lock" % target_path while True: if not os.path.exists(target_path_lock): # Need to create a dummy file, no need to do IO # Hence no open and close business os.mknod(target_path_lock) break time.sleep(0.2) # Ignore if already mounted if os.path.ismount(target_path): logging.debug(logf("Already mounted (2nd try)", mount=target_path)) os.unlink(target_path_lock) return if volume['type'] == 'External': # Try to mount the Host Volume, handle failure if # already mounted mount_glusterfs_with_host(volume['g_volname'], target_path, volume['g_host'], volume['g_options']) os.unlink(target_path_lock) return generate_client_volfile(volume['name']) # Fix the log, so we can check it out later # log_file = "/var/log/gluster/%s.log" % target_path.replace("/", "-") log_file = "/var/log/gluster/gluster.log" cmd = [ GLUSTERFS_CMD, "--process-name", "fuse", "-l", log_file, "--volfile-id", volume['name'], "-f", "%s/%s.client.vol" % (VOLFILES_DIR, volume['name']), target_path ] try: execute(*cmd) except Exception as err: logging.error( logf("error to execute command", volume=volume, cmd=cmd, error=format(err))) os.unlink(target_path_lock) raise err os.unlink(target_path_lock) return
def mount_glusterfs(volume, mountpoint, is_client=False): """Mount Glusterfs Volume""" if volume["type"] == "External": volname = volume['g_volname'] else: volname = volume["name"] # Ignore if already glusterfs process running for that volume if is_gluster_mount_proc_running(volname, mountpoint): logging.debug(logf("Already mounted", mount=mountpoint)) return # Ignore if already mounted if is_gluster_mount_proc_running(volname, mountpoint): logging.debug(logf("Already mounted (2nd try)", mount=mountpoint)) return if not os.path.exists(mountpoint): makedirs(mountpoint) if volume['type'] == 'External': # Try to mount the Host Volume, handle failure if # already mounted with mount_lock: mount_glusterfs_with_host(volume['g_volname'], mountpoint, volume['g_host'], volume['g_options'], is_client) return with mount_lock: generate_client_volfile(volume['name']) # Fix the log, so we can check it out later # log_file = "/var/log/gluster/%s.log" % mountpoint.replace("/", "-") log_file = "/var/log/gluster/gluster.log" cmd = [ GLUSTERFS_CMD, "--process-name", "fuse", "-l", log_file, "--volfile-id", volume['name'], "--fs-display-name", "kadalu:%s" % volume['name'], "-f", "%s/%s.client.vol" % (VOLFILES_DIR, volume['name']), mountpoint ] ## required for 'simple-quota' if not is_client: cmd.extend(["--client-pid", "-14"]) try: execute(*cmd) except CommandException as err: logging.error( logf("error to execute command", volume=volume, cmd=cmd, error=format(err))) raise err return
def create_virtblock_volume(hostvol_mnt, volname, size): """Create virtual block volume""" volhash = get_volname_hash(volname) volpath = get_volume_path(PV_TYPE_VIRTBLOCK, volhash, volname) volpath_full = os.path.join(hostvol_mnt, volpath) logging.debug(logf( "Volume hash", volhash=volhash )) # Check for mount availability before creating virtblock volume retry_errors(os.statvfs, [hostvol_mnt], [ENOTCONN]) # Create a file with required size makedirs(os.path.dirname(volpath_full)) logging.debug(logf( "Created virtblock directory", path=os.path.dirname(volpath) )) if os.path.exists(volpath_full): rand = time.time() logging.info(logf( "Getting 'Create request' on existing file, renaming.", path=volpath_full, random=rand )) os.rename(volpath_full, "%s.%s" % (volpath_full, rand)) volpath_fd = os.open(volpath_full, os.O_CREAT | os.O_RDWR) os.close(volpath_fd) os.truncate(volpath_full, size) logging.debug(logf( "Truncated file to required size", path=volpath, size=size )) # TODO: Multiple FS support based on volume_capability mount option execute(MKFS_XFS_CMD, volpath_full) logging.debug(logf( "Created Filesystem", path=volpath, command=MKFS_XFS_CMD )) save_pv_metadata(hostvol_mnt, volpath, size) return Volume( volname=volname, voltype=PV_TYPE_VIRTBLOCK, volhash=volhash, hostvol=os.path.basename(hostvol_mnt), size=size, volpath=volpath, )
def mount_volume(pvpath, mountpoint, pvtype, fstype=None): """Mount a Volume""" # Need this after kube 1.20.0 makedirs(mountpoint) if pvtype == PV_TYPE_VIRTBLOCK: fstype = "xfs" if fstype is None else fstype execute(MOUNT_CMD, "-t", fstype, pvpath, mountpoint) else: execute(MOUNT_CMD, "--bind", pvpath, mountpoint) os.chmod(mountpoint, 0o777)
def mount_glusterfs(volume, target_path): """Mount Glusterfs Volume""" if not os.path.exists(target_path): makedirs(target_path) # Ignore if already mounted if os.path.ismount(target_path): logging.debug(logf("Already mounted", mount=target_path)) return # Ignore if already mounted if os.path.ismount(target_path): logging.debug(logf("Already mounted (2nd try)", mount=target_path)) return if volume['type'] == 'External': # Try to mount the Host Volume, handle failure if # already mounted with mount_lock: mount_glusterfs_with_host(volume['g_volname'], target_path, volume['g_host'], volume['g_options']) return with mount_lock: generate_client_volfile(volume['name']) # Fix the log, so we can check it out later # log_file = "/var/log/gluster/%s.log" % target_path.replace("/", "-") log_file = "/var/log/gluster/gluster.log" cmd = [ GLUSTERFS_CMD, "--process-name", "fuse", "-l", log_file, "--volfile-id", volume['name'], "-f", "%s/%s.client.vol" % (VOLFILES_DIR, volume['name']), target_path ] try: execute(*cmd) except CommandException as err: logging.error( logf("error to execute command", volume=volume, cmd=cmd, error=format(err))) raise err return
def save_pv_metadata(hostvol_mnt, pvpath, pvsize): """Save PV metadata in info file""" # Create info dir if not exists info_file_path = os.path.join(hostvol_mnt, "info", pvpath) info_file_dir = os.path.dirname(info_file_path) makedirs(info_file_dir) logging.debug( logf("Created metadata directory", metadata_dir=info_file_dir)) with open(info_file_path + ".json", "w") as info_file: info_file.write( json.dumps({ "size": pvsize, "path_prefix": os.path.dirname(pvpath) })) logging.debug(logf( "Metadata saved", metadata_file=info_file_path, ))
def update_virtblock_volume(hostvol_mnt, volname, expansion_requested_pvsize): """Update virtual block volume""" volhash = get_volname_hash(volname) volpath = get_volume_path(PV_TYPE_VIRTBLOCK, volhash, volname) volpath_full = os.path.join(hostvol_mnt, volpath) logging.debug(logf("Volume hash", volhash=volhash)) # Check for mount availability before updating virtblock volume retry_errors(os.statvfs, [hostvol_mnt], [ENOTCONN]) # Update the file with required size makedirs(os.path.dirname(volpath_full)) logging.debug( logf("Updated virtblock directory", path=os.path.dirname(volpath))) volpath_fd = os.open(volpath_full, os.O_CREAT | os.O_RDWR) os.close(volpath_fd) execute("truncate", "-s", expansion_requested_pvsize, volpath_full) logging.debug( logf("Truncated file to required size", path=volpath, size=expansion_requested_pvsize)) # TODO: Multiple FS support based on volume_capability mount option execute(MKFS_XFS_CMD, volpath_full) logging.debug( logf("Created Filesystem", path=volpath, command=MKFS_XFS_CMD)) update_pv_metadata(hostvol_mnt, volpath, expansion_requested_pvsize) return Volume( volname=volname, voltype=PV_TYPE_VIRTBLOCK, volhash=volhash, hostvol=os.path.basename(hostvol_mnt), size=expansion_requested_pvsize, volpath=volpath, )
def mount_volume(pvpath, mountpoint, pvtype, fstype=None): """Mount a Volume""" # Create subvol dir if PV is manually created if not os.path.exists(pvpath): makedirs(pvpath) # TODO: Will losetup survive container reboot? if pvtype == PV_TYPE_RAWBLOCK: # losetup of truncated file cmd = ["losetup", "-f", "--show", pvpath] try: loop, _, _ = execute(*cmd) except CommandException as err: # Better not to create loop devices manually errmsg = "Please check availability of 'losetup' and 'loop' device" logging.error(logf(errmsg, cmd=cmd, error=format(err))) return False # Bind mount loop device to target_path, stage_path may not be needed makedirs(os.path.dirname(mountpoint)) Path(mountpoint).touch(mode=0o777) execute(MOUNT_CMD, "--bind", loop, mountpoint) return True # Need this after kube 1.20.0 makedirs(mountpoint) if pvtype == PV_TYPE_VIRTBLOCK: fstype = "xfs" if fstype is None else fstype execute(MOUNT_CMD, "-t", fstype, pvpath, mountpoint) else: execute(MOUNT_CMD, "--bind", pvpath, mountpoint) os.chmod(mountpoint, 0o777) return True
def create_subdir_volume(hostvol_mnt, volname, size): """Create sub directory Volume""" volhash = get_volname_hash(volname) volpath = get_volume_path(PV_TYPE_SUBVOL, volhash, volname) logging.debug(logf("Volume hash", volhash=volhash)) # Check for mount availability before creating subdir volume retry_errors(os.statvfs, [hostvol_mnt], [ENOTCONN]) # Create a subdir makedirs(os.path.join(hostvol_mnt, volpath)) logging.debug(logf("Created PV directory", pvdir=volpath)) # Write info file so that Brick's quotad sidecar # container picks it up. save_pv_metadata(hostvol_mnt, volpath, size) # Wait for quota set # TODO: Handle Timeout pvsize_buffer = size * 0.05 # 5% pvsize_min = (size - pvsize_buffer) pvsize_max = (size + pvsize_buffer) logging.debug( logf( "Watching df of pv directory", pvdir=volpath, pvsize_buffer=pvsize_buffer, )) #setfattr -n trusted.glusterfs.namespace -v true #setfattr -n trusted.gfs.squota.limit -v size try: retry_errors(os.setxattr, [ os.path.join(hostvol_mnt, volpath), "trusted.glusterfs.namespace", "true".encode() ], [ENOTCONN]) retry_errors(os.setxattr, [ os.path.join(hostvol_mnt, volpath), "trusted.gfs.squota.limit", str(size).encode() ], [ENOTCONN]) # noqa # pylint: disable=broad-except except Exception as err: logging.info( logf("Failed to set quota using simple-quota. Continuing", error=err)) count = 0 while True: count += 1 pvstat = retry_errors(os.statvfs, [os.path.join(hostvol_mnt, volpath)], [ENOTCONN]) volsize = pvstat.f_blocks * pvstat.f_bsize if pvsize_min < volsize < pvsize_max: logging.debug( logf("Matching df output, Quota set successful", volsize=volsize, num_tries=count)) break if count >= 6: logging.warning( logf("Waited for some time, Quota set failed, continuing.", volsize=volsize, num_tries=count)) break time.sleep(1) return Volume( volname=volname, voltype=PV_TYPE_SUBVOL, volhash=volhash, hostvol=os.path.basename(hostvol_mnt), size=size, volpath=volpath, )
def update_subdir_volume(hostvol_mnt, hostvoltype, volname, expansion_requested_pvsize): """Update sub directory Volume""" volhash = get_volname_hash(volname) volpath = get_volume_path(PV_TYPE_SUBVOL, volhash, volname) logging.debug(logf("Volume hash", volhash=volhash)) # Check for mount availability before updating subdir volume retry_errors(os.statvfs, [hostvol_mnt], [ENOTCONN]) # Create a subdir makedirs(os.path.join(hostvol_mnt, volpath)) logging.debug(logf("Updated PV directory", pvdir=volpath)) # Write info file so that Brick's quotad sidecar # container picks it up. update_pv_metadata(hostvol_mnt, volpath, expansion_requested_pvsize) # Wait for quota set # TODO: Handle Timeout pvsize_buffer = expansion_requested_pvsize * 0.05 # 5% pvsize_min = (expansion_requested_pvsize - pvsize_buffer) pvsize_max = (expansion_requested_pvsize + pvsize_buffer) logging.debug( logf( "Watching df of pv directory", pvdir=volpath, pvsize_buffer=pvsize_buffer, )) # Handle this case in calling function if hostvoltype == 'External': return None retry_errors(os.setxattr, [ os.path.join(hostvol_mnt, volpath), "trusted.gfs.squota.limit", str(expansion_requested_pvsize).encode() ], [ENOTCONN]) count = 0 while True: count += 1 pvstat = retry_errors(os.statvfs, [os.path.join(hostvol_mnt, volpath)], [ENOTCONN]) volsize = pvstat.f_blocks * pvstat.f_bsize if pvsize_min < volsize < pvsize_max: logging.debug( logf("Matching df output, Quota update set successful", volsize=volsize, pvsize=expansion_requested_pvsize, num_tries=count)) break if count >= 6: logging.warning( logf( "Waited for some time, Quota update set failed, continuing.", volsize=volsize, pvsize=expansion_requested_pvsize, num_tries=count)) break time.sleep(1) return Volume( volname=volname, voltype=PV_TYPE_SUBVOL, volhash=volhash, hostvol=os.path.basename(hostvol_mnt), size=expansion_requested_pvsize, volpath=volpath, )
def mount_glusterfs_with_host(volname, mountpoint, hosts, options=None, is_client=False): """Mount Glusterfs Volume""" # Ignore if already mounted if is_gluster_mount_proc_running(volname, mountpoint): logging.debug(logf("Already mounted", mount=mountpoint)) return if not os.path.exists(mountpoint): makedirs(mountpoint) log_file = "/var/log/gluster/gluster.log" cmd = [ GLUSTERFS_CMD, "--process-name", "fuse", "-l", "%s" % log_file, "--volfile-id", volname, ] ## on server component we can mount glusterfs with client-pid #if not is_client: # cmd.extend(["--client-pid", "-14"]) for host in hosts.split(','): cmd.extend(["--volfile-server", host]) g_ops = [] if options: for option in options.split(","): g_ops.append(f"--{option}") logging.debug( logf( "glusterfs command", cmd=cmd, opts=g_ops, mountpoint=mountpoint, )) command = cmd + g_ops + [mountpoint] try: execute(*command) except CommandException as excep: if excep.err.find("invalid option") != -1: logging.info( logf( "proceeding without supplied incorrect mount options", options=g_ops, )) command = cmd + [mountpoint] try: execute(*command) except CommandException as excep: logging.info( logf( "mount command failed", cmd=command, error=excep, )) return logging.info(logf( "mount command failed", cmd=command, error=excep, )) return
def mount_glusterfs(volume, mountpoint, storage_options="", is_client=False): """Mount Glusterfs Volume""" if volume["type"] == "External": volname = volume['g_volname'] else: volname = volume["name"] if volume['type'] == 'External': # Try to mount the Host Volume, handle failure if # already mounted with mount_lock: mount_glusterfs_with_host(volname, mountpoint, volume['g_host'], volume['g_options'], is_client) use_gluster_quota = False if (os.path.isfile("/etc/secret-volume/ssh-privatekey") and "SECRET_GLUSTERQUOTA_SSH_USERNAME" in os.environ): use_gluster_quota = True secret_private_key = "/etc/secret-volume/ssh-privatekey" secret_username = os.environ.get('SECRET_GLUSTERQUOTA_SSH_USERNAME', None) # SSH into only first reachable host in volume['g_host'] entry g_host = reachable_host(volume['g_host']) if g_host is None: logging.error(logf("All hosts are not reachable")) return if use_gluster_quota is False: logging.debug(logf("Do not set quota-deem-statfs")) else: logging.debug( logf("Set quota-deem-statfs for gluster directory Quota")) quota_deem_cmd = [ "ssh", "-oStrictHostKeyChecking=no", "-i", "%s" % secret_private_key, "%s@%s" % (secret_username, g_host), "sudo", "gluster", "volume", "set", "%s" % volume['g_volname'], "quota-deem-statfs", "on" ] try: execute(*quota_deem_cmd) except CommandException as err: errmsg = "Unable to set quota-deem-statfs via ssh" logging.error(logf(errmsg, error=err)) raise err return mountpoint generate_client_volfile(volname) client_volfile_path = os.path.join(VOLFILES_DIR, "%s.client.vol" % volname) if storage_options != "": # Construct 'dict' from passed storage-options in 'str' storage_options = storage_options_parse(storage_options) # Keep the default volfile untouched tmp_volfile_path = tempfile.mkstemp()[1] shutil.copy(client_volfile_path, tmp_volfile_path) # Parse the client-volfile, update passed storage-options & save parsed_client_volfile_path = Volfile.parse(tmp_volfile_path) parsed_client_volfile_path.update_options_by_type(storage_options) parsed_client_volfile_path.save() # Sort storage-options and generate hash storage_options_hash = get_storage_options_hash( json.dumps(storage_options, sort_keys=True)) # Rename mountpoint & client volfile path with hash mountpoint = mountpoint + "_" + storage_options_hash new_client_volfile_path = os.path.join( VOLFILES_DIR, "%s_%s.client.vol" % (volname, storage_options_hash)) os.rename(tmp_volfile_path, new_client_volfile_path) client_volfile_path = new_client_volfile_path # Ignore if already glusterfs process running for that volume if is_gluster_mount_proc_running(volname, mountpoint): reload_glusterfs(volume) logging.debug(logf("Already mounted", mount=mountpoint)) return mountpoint # Ignore if already mounted if is_gluster_mount_proc_running(volname, mountpoint): reload_glusterfs(volume) logging.debug(logf("Already mounted (2nd try)", mount=mountpoint)) return mountpoint if not os.path.exists(mountpoint): makedirs(mountpoint) with mount_lock: # Fix the log, so we can check it out later # log_file = "/var/log/gluster/%s.log" % mountpoint.replace("/", "-") log_file = "/var/log/gluster/gluster.log" cmd = [ GLUSTERFS_CMD, "--process-name", "fuse", "-l", log_file, "--volfile-id", volname, "--fs-display-name", "kadalu:%s" % volname, "-f", client_volfile_path, mountpoint ] ## required for 'simple-quota' if not is_client: cmd.extend(["--client-pid", "-14"]) try: (_, err, pid) = execute(*cmd) VOL_DATA[volname]["pid"] = pid except CommandException as err: logging.error( logf("error to execute command", volume=volume, cmd=cmd, error=format(err))) raise err return mountpoint