class Node(csi.NodeServicer, Identity):
STAGE_RESP = types.StageResp()
UNSTAGE_RESP = types.UnstageResp()
NODE_PUBLISH_RESP = types.NodePublishResp()
NODE_UNPUBLISH_RESP = types.NodeUnpublishResp()
def __init__(self,
server,
persistence_config=None,
cinderlib_config=None,
node_id=None,
storage_nw_ip=None,
**kwargs):
if persistence_config:
cinderlib_config['fail_on_missing_backend'] = False
cinderlib.setup(persistence_config=persistence_config,
**cinderlib_config)
Identity.__init__(self, server, cinderlib_config)
node_id = node_id or socket.getfqdn()
self.node_id = types.IdResp(node_id=node_id)
self.node_info = NodeInfo.set(node_id, storage_nw_ip)
csi.add_NodeServicer_to_server(self, server)
def _get_split_file(self, filename):
with open(filename) as f:
result = [line.split() for line in f.read().split('\n') if line]
return result
def _get_mountinfo(self):
return self._get_split_file('/proc/self/mountinfo')
def _vol_private_location(self, volume_id):
private_bind = os.path.join(os.getcwd(), volume_id)
return private_bind
def _get_mount(self, private_bind):
mounts = self._get_split_file('/proc/self/mounts')
result = [mount for mount in mounts if mount[0] == private_bind]
return result
def _get_device(self, path):
for line in self._get_mountinfo():
if line[4] == path:
return line[9] if line[9].startswith('/') else line[3]
return None
def _get_vol_device(self, volume_id):
private_bind = self._vol_private_location(volume_id)
device = self._get_device(private_bind)
return device, private_bind
def _format_device(self, fs_type, device, context):
# We don't use the util-linux Python library to reduce dependencies
stdout, stderr = self.sudo('lsblk',
'-nlfoFSTYPE',
device,
retries=5,
errors=[1, 32],
delay=2)
fs_types = filter(None, stdout.split())
if fs_types:
if fs_types[0] == fs_type:
return
context.abort(
grpc.StatusCode.ALREADY_EXISTS,
'Cannot stage filesystem %s on device that '
'already has filesystem %s' % (fs_type, fs_types[0]))
cmd = [self.MKFS + fs_type]
cmd.extend(self.MKFS_ARGS.get(fs_type, self.DEFAULT_MKFS_ARGS))
cmd.append(device)
self.sudo(*cmd)
def _check_mount_exists(self, capability, private_bind, target, context):
mounts = self._get_mount(private_bind)
if mounts:
if target != mounts[0][1]:
context.abort(
grpc.StatusCode.ALREADY_EXISTS,
'Filesystem already mounted on %s' % mounts[0][1])
requested_flags = set(capability.mount.mount_flags or [])
missing_flags = requested_flags.difference(mounts[0][3].split(','))
if missing_flags:
context.abort(
grpc.StatusCode.ALREADY_EXISTS,
'Already mounted with different flags (%s)' %
missing_flags)
return True
return False
def _mount(self, fs_type, mount_flags, private_bind, target):
# Mount must only be called if it's already not mounted
# We don't use the util-linux Python library to reduce dependencies
command = ['mount', '-t', fs_type]
if mount_flags:
command.append('-o')
command.append(','.join(mount_flags))
command.append(private_bind)
command.append(target)
self.sudo(*command)
def _check_path(self, request, context, is_staging):
is_block = request.volume_capability.HasField('block')
attr_name = 'staging_target_path' if is_staging else 'target_path'
path = getattr(request, attr_name)
try:
st_mode = os.stat(path).st_mode
except OSError as exc:
context.abort(grpc.StatusCode.INVALID_ARGUMENT,
'Invalid %s path: %s' % (attr_name, exc))
if ((is_block and stat.S_ISBLK(st_mode) or stat.S_ISREG(st_mode))
or (not is_block and stat.S_ISDIR(st_mode))):
return path, is_block
context.abort(grpc.StatusCode.INVALID_ARGUMENT,
'Invalid existing %s' % attr_name)
@debuggable
@logrpc
@require('volume_id', 'staging_target_path', 'volume_capability')
@Worker.unique
def NodeStageVolume(self, request, context):
vol = self._get_vol(request.volume_id)
if not vol:
context.abort(grpc.StatusCode.NOT_FOUND,
'Volume %s does not exist' % request.volume_id)
self._validate_capabilities([request.volume_capability], context)
target, is_block = self._check_path(request, context, is_staging=True)
device, private_bind = self._get_vol_device(vol.id)
if not device:
# For now we don't really require the publish_info, since we share
# the persistence storage, but if we would need to deserialize it
# with json.loads from key 'connection_info'
conn = vol.connections[0]
# Some slow systems may take a while to detect the multipath so we
# retry the attach. Since we don't disconnect this will go fast
# through the login phase.
for i in range(MULTIPATH_FIND_RETRIES):
conn.attach()
if not conn.use_multipath or conn.path.startswith('/dev/dm'):
break
sys.stdout.write('Retrying to get a multipath\n')
# Create the private bind file
open(private_bind, 'a').close()
# TODO(geguileo): make path for private binds configurable
self.sudo('mount', '--bind', conn.path, private_bind)
device = conn.path
if is_block:
# Avoid multiple binds if CO incorrectly called us twice
device = self._get_device(target)
if not device:
# TODO(geguileo): Add support for NFS/QCOW2
self.sudo('mount', '--bind', private_bind, target)
else:
if not self._check_mount_exists(request.volume_capability,
private_bind, target, context):
fs_type = (request.volume_capability.mount.fs_type
or DEFAULT_MOUNT_FS)
self._format_device(fs_type, private_bind, context)
self._mount(fs_type,
request.volume_capability.mount.mount_flags,
private_bind, target)
return self.STAGE_RESP
@debuggable
@logrpc
@require('volume_id', 'staging_target_path')
@Worker.unique
def NodeUnstageVolume(self, request, context):
# TODO(geguileo): Add support for NFS/QCOW2
vol = self._get_vol(request.volume_id)
if not vol:
context.abort(grpc.StatusCode.NOT_FOUND,
'Volume %s does not exist' % request.volume_id)
device, private_bind = self._get_vol_device(vol.id)
# If it's not already unstaged
if device:
count = 0
for line in self._get_mountinfo():
if line[3] in (device, private_bind):
count += 1
if self._get_mount(private_bind):
count += 1
# If the volume is still in use we cannot unstage (one use is for
# our private volume reference and the other for staging path
if count > 2:
context.abort(grpc.StatusCode.ABORTED,
'Operation pending for volume')
if count == 2:
self.sudo('umount', request.staging_target_path, retries=4)
if count > 0:
self.sudo('umount', private_bind, retries=4)
os.remove(private_bind)
conn = vol.connections[0]
conn.detach()
return self.UNSTAGE_RESP
@debuggable
@logrpc
@require('volume_id', 'staging_target_path', 'target_path',
'volume_capability')
@Worker.unique
def NodePublishVolume(self, request, context):
self._validate_capabilities([request.volume_capability], context)
staging_target, is_block = self._check_path(request,
context,
is_staging=True)
device, private_bind = self._get_vol_device(request.volume_id)
error = (not device
or (is_block and not self._get_device(staging_target))
or (not is_block and not self._check_mount_exists(
request.volume_capability, private_bind, staging_target,
context)))
if error:
context.abort(grpc.StatusCode.FAILED_PRECONDITION,
'Staging was not been successfully called')
target, is_block = self._check_path(request, context, is_staging=False)
# TODO(geguileo): Add support for modes, etc.
# Check if it's already published
device = self._get_device(target)
volume_device, private_bind = self._get_vol_device(request.volume_id)
if device in (volume_device, staging_target, private_bind):
return self.NODE_PUBLISH_RESP
# TODO(geguileo): Check how many are mounted and fail if > 0
# If not published bind it
self.sudo('mount', '--bind', staging_target, target)
return self.NODE_PUBLISH_RESP
@debuggable
@logrpc
@require('volume_id', 'target_path')
@Worker.unique
def NodeUnpublishVolume(self, request, context):
device = self._get_device(request.target_path)
if device:
self.sudo('umount', request.target_path, retries=4)
return self.NODE_UNPUBLISH_RESP
@debuggable
@logrpc
def NodeGetId(self, request, context):
return self.node_id
@debuggable
@logrpc
def NodeGetCapabilities(self, request, context):
rpc = types.NodeCapabilityType.STAGE_UNSTAGE_VOLUME
capabilities = [types.NodeCapability(rpc=types.NodeRPC(type=rpc))]
return types.NodeCapabilityResp(capabilities=capabilities)
class Node(csi.NodeServicer, Identity):
STAGE_RESP = types.StageResp()
UNSTAGE_RESP = types.UnstageResp()
NODE_PUBLISH_RESP = types.NodePublishResp()
NODE_UNPUBLISH_RESP = types.NodeUnpublishResp()
def __init__(self,
server,
persistence_config=None,
node_id=None,
storage_nw_ip=None,
**kwargs):
if persistence_config:
self.persistence = persistence.setup(persistence_config)
# TODO(geguileo): Make Node only service work, which may require
# modifications to cinderlib or faking the Backend object, since
# all objects set the backend field on initialization.
# cinderlib.objects.Object.setup(self.persistence, ...)
node_id = node_id or socket.getfqdn()
self.node_id = types.IdResp(node_id=node_id)
self.node_info = NodeInfo.set(node_id, storage_nw_ip)
Identity.__init__(self, server)
csi.add_NodeServicer_to_server(self, server)
def _get_mountinfo(self):
with open('/proc/self/mountinfo') as f:
mountinfo = [line.split() for line in f.read().split('\n') if line]
return mountinfo
def _vol_private_location(self, volume_id):
private_bind = os.path.join(os.getcwd(), volume_id)
return private_bind
def _get_device(self, path):
for line in self._get_mountinfo():
if line[4] == path:
return line[3]
return None
def _get_vol_device(self, volume_id):
private_bind = self._vol_private_location(volume_id)
device = self._get_device(private_bind)
return device, private_bind
def NodeStageVolume(self, request, context):
vol = self._get_vol(request.volume_id)
if not vol:
context.abort(grpc.StatusCode.NOT_FOUND,
'Volume %s does not exist' % request.volume_id)
# TODO(geguileo): Check capabilities
# TODO(geguileo): Check that the provided staging_target_path is an
# existing file for block or directory for filesystems
device, private_bind = self._get_vol_device(vol.id)
if not device:
# For now we don't really require the publish_info, since we share
# the persistence storage, but if we would need to deserialize it
# with json.loads from key 'connection_info'
conn = vol.connections[0]
conn.attach()
# Create the private bind file
open(private_bind, 'a').close()
# TODO(geguileo): make path for private binds configurable
self.sudo('mount', '--bind', conn.path, private_bind)
# If CO did something wrong and called us twice avoid multiple binds
device = self._get_device(request.staging_target_path)
if not device:
# TODO(geguileo): Add support for NFS/QCOW2
self.sudo('mount', '--bind', private_bind,
request.staging_target_path)
return self.STAGE_RESP
def NodeUnstageVolume(self, request, context):
# TODO(geguileo): Add support for NFS/QCOW2
vol = self._get_vol(request.volume_id)
if not vol:
context.abort(grpc.StatusCode.NOT_FOUND,
'Volume %s does not exist' % request.volume_id)
device, private_bind = self._get_vol_device(vol.id)
# If it's not already unstaged
if device:
count = 0
for line in self._get_mountinfo():
if line[3] in (device, private_bind):
count += 1
# If the volume is still in use we cannot unstage (one use is for
# our private volume reference and the other for staging path
if count > 2:
context.abort(grpc.StatusCode.ABORTED,
'Operation pending for volume')
conn = vol.connections[0]
if count == 2:
self.sudo('umount', request.staging_target_path)
conn.detach()
if count > 0:
self.sudo('umount', private_bind)
os.remove(private_bind)
return self.UNSTAGE_RESP
def NodePublishVolume(self, request, context):
# TODO(geguileo): Check if staging_target_path is passed and exists
# TODO(geguileo): Add support for modes, etc.
# Check if it's already published
device = self._get_device(request.target_path)
volume_device, private_bind = self._get_vol_device(request.volume_id)
if device in (volume_device, request.staging_target_path):
return self.NODE_PUBLISH_RESP
# TODO(geguileo): Check how many are mounted and fail if > 0
# If not published bind it
self.sudo('mount', '--bind', request.staging_target_path,
request.target_path)
return self.NODE_PUBLISH_RESP
def NodeUnpublishVolume(self, request, context):
device = self._get_device(request.target_path)
if device:
self.sudo('umount', request.target_path)
return self.NODE_UNPUBLISH_RESP
def NodeGetId(self, request, context):
return self.node_id
def NodeGetCapabilities(self, request, context):
rpc = types.NodeCapabilityType.STAGE_UNSTAGE_VOLUME
capabilities = [types.NodeCapability(rpc=types.NodeRPC(type=rpc))]
return types.NodeCapabilityResp(capabilities=capabilities)