def _get_available_capacity(self):
"""Calculate available space on path."""
try:
(out, __) = self._hdfs_execute(self._hdfs_bin, 'dfsadmin',
'-report')
except exception.ProcessExecutionError as e:
msg = (_('Failed to check available capacity for hdfs.'
'Error: %(excmsg)s.') % {
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
lines = out.splitlines()
try:
total = int(lines[1].split()[2])
free = int(lines[2].split()[2])
except (IndexError, ValueError) as e:
msg = (_('Failed to get hdfs capacity info. '
'Error: %(excmsg)s.') % {
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
return total, free
def _create_share(self, share):
"""Creates a share."""
if share['share_proto'].lower() != 'hdfs':
msg = _('Only HDFS protocol supported!')
LOG.error(msg)
raise exception.HDFSException(msg)
share_dir = '/' + share['name']
try:
self._hdfs_execute(self._hdfs_bin, 'dfs', '-mkdir', share_dir)
except exception.ProcessExecutionError as e:
msg = (_('Failed to create directory in hdfs for the '
'share %(sharename)s. Error: %(excmsg)s.') % {
'sharename': share['name'],
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
# set share size
self._set_share_size(share)
try:
self._hdfs_execute(self._hdfs_bin, 'dfsadmin', '-allowSnapshot',
share_dir)
except exception.ProcessExecutionError as e:
msg = (_('Failed to allow snapshot for the '
'share %(sharename)s. Error: %(excmsg)s.') % {
'sharename': share['name'],
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
def check_for_setup_error(self):
"""Return an error if the prerequisites are met."""
if not self.configuration.hdfs_namenode_ip:
msg = _('Not specify the hdfs cluster yet! '
'Add the ip of hdfs namenode in the '
'hdfs_namenode_ip configuration parameter.')
LOG.error(msg)
raise exception.HDFSException(msg)
if not self._check_hdfs_state():
msg = _('HDFS is not in healthy state.')
LOG.error(msg)
raise exception.HDFSException(msg)
def create_share_from_snapshot(self,
context,
share,
snapshot,
share_server=None):
"""Creates a snapshot."""
self._create_share(share)
share_path = '/' + share['name']
snapshot_path = self._get_snapshot_path(snapshot)
try:
# check if the directory is empty
(out, __) = self._hdfs_execute(self._hdfs_bin, 'dfs', '-ls',
snapshot_path)
# only copy files when the snapshot directory is not empty
if out:
copy_path = snapshot_path + "/*"
cmd = [self._hdfs_bin, 'dfs', '-cp', copy_path, share_path]
self._hdfs_execute(*cmd)
except exception.ProcessExecutionError as e:
msg = (_('Failed to create share %(sharename)s from '
'snapshot %(snapshotname)s. Error: %(excmsg)s.') % {
'sharename': share['name'],
'snapshotname': snapshot['name'],
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
return self._get_share_path(share)
def _get_hdfs_bin_path(self):
try:
(out, __) = self._hdfs_execute('locate', '/bin/hdfs')
except exception.ProcessExecutionError as e:
msg = (_('Can not get the execution path of hdfs. '
'Error: %(excmsg)s.') % {
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
lines = out.splitlines()
if lines and lines[0].endswith('/bin/hdfs'):
return lines[0]
else:
msg = _('Can not get the execution path of hdfs.')
LOG.error(msg)
raise exception.HDFSException(msg)
def _check_hdfs_state(self):
try:
(out, __) = self._hdfs_execute(self._hdfs_bin, 'fsck', '/')
except exception.ProcessExecutionError as e:
msg = _('Failed to check the utility of hdfs.')
LOG.error(msg)
raise exception.HDFSException(msg)
lines = out.splitlines()
try:
hdfs_state = lines[1].split()[1]
except (IndexError, ValueError) as e:
msg = (_('Failed to check hdfs state. Error: %(excmsg)s.') % {
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
if hdfs_state.upper() != 'HEALTHY':
return False
return True
def _check_hdfs_state(self):
try:
(out, __) = self._hdfs_execute(self._hdfs_bin, 'fsck', '/')
except exception.ProcessExecutionError as e:
msg = (_('Failed to check hdfs state. Error: %(excmsg)s.') %
{'excmsg': six.text_type(e)})
LOG.error(msg)
raise exception.HDFSException(msg)
if 'HEALTHY' in out:
return True
else:
return False
def delete_share(self, context, share, share_server=None):
"""Deletes share storage."""
share_dir = '/' + share['name']
cmd = [self._hdfs_bin, 'dfs', '-rm', '-r', share_dir]
try:
self._hdfs_execute(*cmd)
except exception.ProcessExecutionError as e:
msg = (_('Failed to delete share %(sharename)s. '
'Error: %(excmsg)s.') %
{'sharename': share['name'],
'excmsg': six.text_type(e)})
LOG.error(msg)
raise exception.HDFSException(msg)
def delete_snapshot(self, context, snapshot, share_server=None):
"""Deletes a snapshot."""
share_dir = '/' + snapshot['share_name']
cmd = [self._hdfs_bin, 'dfs', '-deleteSnapshot',
share_dir, snapshot['name']]
try:
self._hdfs_execute(*cmd)
except exception.ProcessExecutionError as e:
msg = (_('Failed to delete snapshot %(snapshotname)s. '
'Error: %(excmsg)s.') %
{'snapshotname': snapshot['name'],
'excmsg': six.text_type(e)})
LOG.error(msg)
raise exception.HDFSException(msg)
def deny_access(self, context, share, access, share_server=None):
"""Denies the access to the share for a given user."""
share_dir = '/' + share['name']
access_name = ':'.join([access['access_type'], access['access_to']])
cmd = [self._hdfs_bin, 'dfs', '-setfacl', '-x', '-R',
access_name, share_dir]
try:
(__, out) = self._hdfs_execute(*cmd, check_exit_code=True)
except exception.ProcessExecutionError as e:
msg = (_('Failed to deny ACL of share %(sharename)s for '
'user: %(username)s'
'Error: %(excmsg)s.') %
{'sharename': share['name'],
'username': access['access_to'],
'excmsg': six.text_type(e)})
LOG.error(msg)
raise exception.HDFSException(msg)
def _set_share_size(self, share, size=None):
share_dir = '/' + share['name']
if not size:
sizestr = six.text_type(share['size']) + 'g'
else:
sizestr = six.text_type(size) + 'g'
try:
self._hdfs_execute(self._hdfs_bin, 'dfsadmin',
'-setSpaceQuota', sizestr, share_dir)
except exception.ProcessExecutionError as e:
msg = (_('Failed to set space quota for the '
'share %(sharename)s. Error: %(excmsg)s.') %
{'sharename': share['name'],
'excmsg': six.text_type(e)})
LOG.error(msg)
raise exception.HDFSException(msg)
def _run_ssh(self, host, cmd_list, check_exit_code=False):
command = ' '.join(pipes.quote(cmd_arg) for cmd_arg in cmd_list)
connection = self.ssh_connections.get(host)
if not connection:
hdfs_ssh_name = self.configuration.hdfs_ssh_name
password = self.configuration.hdfs_ssh_pw
privatekey = self.configuration.hdfs_ssh_private_key
hdfs_ssh_port = self.configuration.hdfs_ssh_port
ssh_conn_timeout = self.configuration.ssh_conn_timeout
min_size = self.configuration.ssh_min_pool_conn
max_size = self.configuration.ssh_max_pool_conn
ssh_pool = utils.SSHPool(host,
hdfs_ssh_port,
ssh_conn_timeout,
hdfs_ssh_name,
password=password,
privatekey=privatekey,
min_size=min_size,
max_size=max_size)
ssh = ssh_pool.create()
self.ssh_connections[host] = (ssh_pool, ssh)
else:
ssh_pool, ssh = connection
if not ssh.get_transport().is_active():
ssh_pool.remove(ssh)
ssh = ssh_pool.create()
self.ssh_connections[host] = (ssh_pool, ssh)
try:
return processutils.ssh_execute(ssh,
command,
check_exit_code=check_exit_code)
except Exception as e:
msg = (_('Error running SSH command: %(cmd)s. '
'Error: %(excmsg)s.') % {
'cmd': command,
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
def allow_access(self, context, share, access, share_server=None):
"""Allows access to the share for a given user."""
if access['access_type'] != 'user':
msg = _("Only 'user' access type allowed!")
LOG.error(msg)
raise exception.InvalidShareAccess(msg)
# Note(jun): For directories in HDFS, the x permission is
# required to access a child of the directory.
if access['access_level'] == 'rw':
access_level = 'rwx'
elif access['access_level'] == 'ro':
access_level = 'r-x'
else:
msg = (_('The access level %(accesslevel)s was unsupported.') % {
'accesslevel': access['access_level']
})
LOG.error(msg)
raise exception.InvalidShareAccess(msg)
share_dir = '/' + share['name']
user_access = ':'.join(
[access['access_type'], access['access_to'], access_level])
cmd = [
self._hdfs_bin, 'dfs', '-setfacl', '-m', '-R', user_access,
share_dir
]
try:
(__, out) = self._hdfs_execute(*cmd, check_exit_code=True)
except exception.ProcessExecutionError as e:
msg = (_('Failed to set ACL of share %(sharename)s for '
'user: %(username)s'
'Error: %(excmsg)s.') % {
'sharename': share['name'],
'username': access['access_to'],
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
def create_share_from_snapshot(self,
context,
share,
snapshot,
share_server=None):
"""Creates a snapshot."""
self._create_share(share)
share_path = '/' + share['name']
snapshot_path = self._get_snapshot_path(snapshot)
cmd = [self._hdfs_bin, 'dfs', '-cp', '-r', snapshot_path, share_path]
try:
self._hdfs_execute(*cmd)
except exception.ProcessExecutionError as e:
msg = (_('Failed to create share %(sharename)s from '
'snapshot %(snapshotname)s. Error: %(excmsg)s.') % {
'sharename': share['name'],
'snapshotname': snapshot['name'],
'excmsg': six.text_type(e)
})
LOG.error(msg)
raise exception.HDFSException(msg)
return self._get_share_path(share)
