Exemplo n.º 1
0
class HdfsFileManager(FileManagerBase):
    """A wrapper of snakebite client."""

    def can_handle(self, path):
        return path.startswith('hdfs://')

    def __init__(self):
        self._client = AutoConfigClient()

    def ls(self, path: str, recursive=False) -> List[File]:
        files = []
        for file in self._client.ls([path], recurse=recursive):
            if file['file_type'] == 'f':
                files.append(File(
                    path=file['path'],
                    size=file['length']))
        return files

    def move(self, source: str, destination: str) -> bool:
        return len(list(self._client.rename([source], destination))) > 0

    def remove(self, path: str) -> bool:
        return len(list(self._client.delete([path]))) > 0

    def copy(self, source: str, destination: str) -> bool:
        # TODO
        raise NotImplementedError()

    def mkdir(self, path: str) -> bool:
        return next(self._client.mkdir([path], create_parent=True))\
            .get('result')
def mkdir(hdfs_path, create_parent=False, mode=0755):
    """
    paths (list of strings) : Paths to create
    create_parent (boolean) : Also create the parent directories
    mode (int) : Mode the directory should be created with
    Returns:
    String mkdir result as json
    """
    client = AutoConfigClient()

    return list(client.mkdir([hdfs_path], create_parent, mode))
logger.info("Getting user list from /etc/passwd and ldap")

# get a sorted user list from the passwd directory (file+ldap)
user_list = sorted(pwd.getpwall(), key=lambda tup: tup[0])

for user in user_list:
    username = user.pw_name
    userdir = "/user/" + username
    if user.pw_uid <= 500:
        continue
    if user.pw_uid >= 65534:
        continue
    if client.test(userdir, exists=True):
        logger.debug("User exists " +  username)
    else:
        logger.info("username doesn't exist " + username + "; Creating")
        if list(client.mkdir([userdir]))[0]['result']:
            logger.info("Created " + userdir)
            if list(client.chown([userdir], username))[0]['result']:
                logger.info("Chowning userdir to " + username)
            else:
                logger.warn("Could not chown to user" + username)

            if client.test(userdir, exists=True):
                logger.info("... created")
        else:
            logger.warn("Could not create /user/" + username)


logger.info("finished.")