def _get_available_capacity(self): """Calculate available space on path.""" try: (out, __) = self._hdfs_execute(self._hdfs_bin, 'dfsadmin', '-report') except exception.ProcessExecutionError as e: msg = (_('Failed to check available capacity for hdfs.' 'Error: %(excmsg)s.') % { 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) lines = out.splitlines() try: total = int(lines[1].split()[2]) free = int(lines[2].split()[2]) except (IndexError, ValueError) as e: msg = (_('Failed to get hdfs capacity info. ' 'Error: %(excmsg)s.') % { 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) return total, free
def _create_share(self, share): """Creates a share.""" if share['share_proto'].lower() != 'hdfs': msg = _('Only HDFS protocol supported!') LOG.error(msg) raise exception.HDFSException(msg) share_dir = '/' + share['name'] try: self._hdfs_execute(self._hdfs_bin, 'dfs', '-mkdir', share_dir) except exception.ProcessExecutionError as e: msg = (_('Failed to create directory in hdfs for the ' 'share %(sharename)s. Error: %(excmsg)s.') % { 'sharename': share['name'], 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) # set share size self._set_share_size(share) try: self._hdfs_execute(self._hdfs_bin, 'dfsadmin', '-allowSnapshot', share_dir) except exception.ProcessExecutionError as e: msg = (_('Failed to allow snapshot for the ' 'share %(sharename)s. Error: %(excmsg)s.') % { 'sharename': share['name'], 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg)
def check_for_setup_error(self): """Return an error if the prerequisites are met.""" if not self.configuration.hdfs_namenode_ip: msg = _('Not specify the hdfs cluster yet! ' 'Add the ip of hdfs namenode in the ' 'hdfs_namenode_ip configuration parameter.') LOG.error(msg) raise exception.HDFSException(msg) if not self._check_hdfs_state(): msg = _('HDFS is not in healthy state.') LOG.error(msg) raise exception.HDFSException(msg)
def create_share_from_snapshot(self, context, share, snapshot, share_server=None): """Creates a snapshot.""" self._create_share(share) share_path = '/' + share['name'] snapshot_path = self._get_snapshot_path(snapshot) try: # check if the directory is empty (out, __) = self._hdfs_execute(self._hdfs_bin, 'dfs', '-ls', snapshot_path) # only copy files when the snapshot directory is not empty if out: copy_path = snapshot_path + "/*" cmd = [self._hdfs_bin, 'dfs', '-cp', copy_path, share_path] self._hdfs_execute(*cmd) except exception.ProcessExecutionError as e: msg = (_('Failed to create share %(sharename)s from ' 'snapshot %(snapshotname)s. Error: %(excmsg)s.') % { 'sharename': share['name'], 'snapshotname': snapshot['name'], 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) return self._get_share_path(share)
def _get_hdfs_bin_path(self): try: (out, __) = self._hdfs_execute('locate', '/bin/hdfs') except exception.ProcessExecutionError as e: msg = (_('Can not get the execution path of hdfs. ' 'Error: %(excmsg)s.') % { 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) lines = out.splitlines() if lines and lines[0].endswith('/bin/hdfs'): return lines[0] else: msg = _('Can not get the execution path of hdfs.') LOG.error(msg) raise exception.HDFSException(msg)
def _check_hdfs_state(self): try: (out, __) = self._hdfs_execute(self._hdfs_bin, 'fsck', '/') except exception.ProcessExecutionError as e: msg = _('Failed to check the utility of hdfs.') LOG.error(msg) raise exception.HDFSException(msg) lines = out.splitlines() try: hdfs_state = lines[1].split()[1] except (IndexError, ValueError) as e: msg = (_('Failed to check hdfs state. Error: %(excmsg)s.') % { 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) if hdfs_state.upper() != 'HEALTHY': return False return True
def _check_hdfs_state(self): try: (out, __) = self._hdfs_execute(self._hdfs_bin, 'fsck', '/') except exception.ProcessExecutionError as e: msg = (_('Failed to check hdfs state. Error: %(excmsg)s.') % {'excmsg': six.text_type(e)}) LOG.error(msg) raise exception.HDFSException(msg) if 'HEALTHY' in out: return True else: return False
def delete_share(self, context, share, share_server=None): """Deletes share storage.""" share_dir = '/' + share['name'] cmd = [self._hdfs_bin, 'dfs', '-rm', '-r', share_dir] try: self._hdfs_execute(*cmd) except exception.ProcessExecutionError as e: msg = (_('Failed to delete share %(sharename)s. ' 'Error: %(excmsg)s.') % {'sharename': share['name'], 'excmsg': six.text_type(e)}) LOG.error(msg) raise exception.HDFSException(msg)
def delete_snapshot(self, context, snapshot, share_server=None): """Deletes a snapshot.""" share_dir = '/' + snapshot['share_name'] cmd = [self._hdfs_bin, 'dfs', '-deleteSnapshot', share_dir, snapshot['name']] try: self._hdfs_execute(*cmd) except exception.ProcessExecutionError as e: msg = (_('Failed to delete snapshot %(snapshotname)s. ' 'Error: %(excmsg)s.') % {'snapshotname': snapshot['name'], 'excmsg': six.text_type(e)}) LOG.error(msg) raise exception.HDFSException(msg)
def deny_access(self, context, share, access, share_server=None): """Denies the access to the share for a given user.""" share_dir = '/' + share['name'] access_name = ':'.join([access['access_type'], access['access_to']]) cmd = [self._hdfs_bin, 'dfs', '-setfacl', '-x', '-R', access_name, share_dir] try: (__, out) = self._hdfs_execute(*cmd, check_exit_code=True) except exception.ProcessExecutionError as e: msg = (_('Failed to deny ACL of share %(sharename)s for ' 'user: %(username)s' 'Error: %(excmsg)s.') % {'sharename': share['name'], 'username': access['access_to'], 'excmsg': six.text_type(e)}) LOG.error(msg) raise exception.HDFSException(msg)
def _set_share_size(self, share, size=None): share_dir = '/' + share['name'] if not size: sizestr = six.text_type(share['size']) + 'g' else: sizestr = six.text_type(size) + 'g' try: self._hdfs_execute(self._hdfs_bin, 'dfsadmin', '-setSpaceQuota', sizestr, share_dir) except exception.ProcessExecutionError as e: msg = (_('Failed to set space quota for the ' 'share %(sharename)s. Error: %(excmsg)s.') % {'sharename': share['name'], 'excmsg': six.text_type(e)}) LOG.error(msg) raise exception.HDFSException(msg)
def _run_ssh(self, host, cmd_list, check_exit_code=False): command = ' '.join(pipes.quote(cmd_arg) for cmd_arg in cmd_list) connection = self.ssh_connections.get(host) if not connection: hdfs_ssh_name = self.configuration.hdfs_ssh_name password = self.configuration.hdfs_ssh_pw privatekey = self.configuration.hdfs_ssh_private_key hdfs_ssh_port = self.configuration.hdfs_ssh_port ssh_conn_timeout = self.configuration.ssh_conn_timeout min_size = self.configuration.ssh_min_pool_conn max_size = self.configuration.ssh_max_pool_conn ssh_pool = utils.SSHPool(host, hdfs_ssh_port, ssh_conn_timeout, hdfs_ssh_name, password=password, privatekey=privatekey, min_size=min_size, max_size=max_size) ssh = ssh_pool.create() self.ssh_connections[host] = (ssh_pool, ssh) else: ssh_pool, ssh = connection if not ssh.get_transport().is_active(): ssh_pool.remove(ssh) ssh = ssh_pool.create() self.ssh_connections[host] = (ssh_pool, ssh) try: return processutils.ssh_execute(ssh, command, check_exit_code=check_exit_code) except Exception as e: msg = (_('Error running SSH command: %(cmd)s. ' 'Error: %(excmsg)s.') % { 'cmd': command, 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg)
def allow_access(self, context, share, access, share_server=None): """Allows access to the share for a given user.""" if access['access_type'] != 'user': msg = _("Only 'user' access type allowed!") LOG.error(msg) raise exception.InvalidShareAccess(msg) # Note(jun): For directories in HDFS, the x permission is # required to access a child of the directory. if access['access_level'] == 'rw': access_level = 'rwx' elif access['access_level'] == 'ro': access_level = 'r-x' else: msg = (_('The access level %(accesslevel)s was unsupported.') % { 'accesslevel': access['access_level'] }) LOG.error(msg) raise exception.InvalidShareAccess(msg) share_dir = '/' + share['name'] user_access = ':'.join( [access['access_type'], access['access_to'], access_level]) cmd = [ self._hdfs_bin, 'dfs', '-setfacl', '-m', '-R', user_access, share_dir ] try: (__, out) = self._hdfs_execute(*cmd, check_exit_code=True) except exception.ProcessExecutionError as e: msg = (_('Failed to set ACL of share %(sharename)s for ' 'user: %(username)s' 'Error: %(excmsg)s.') % { 'sharename': share['name'], 'username': access['access_to'], 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg)
def create_share_from_snapshot(self, context, share, snapshot, share_server=None): """Creates a snapshot.""" self._create_share(share) share_path = '/' + share['name'] snapshot_path = self._get_snapshot_path(snapshot) cmd = [self._hdfs_bin, 'dfs', '-cp', '-r', snapshot_path, share_path] try: self._hdfs_execute(*cmd) except exception.ProcessExecutionError as e: msg = (_('Failed to create share %(sharename)s from ' 'snapshot %(snapshotname)s. Error: %(excmsg)s.') % { 'sharename': share['name'], 'snapshotname': snapshot['name'], 'excmsg': six.text_type(e) }) LOG.error(msg) raise exception.HDFSException(msg) return self._get_share_path(share)