class HdfsFileManager(FileManagerBase):
"""A wrapper of snakebite client."""
def can_handle(self, path):
return path.startswith('hdfs://')
def __init__(self):
self._client = AutoConfigClient()
def ls(self, path: str, recursive=False) -> List[File]:
files = []
for file in self._client.ls([path], recurse=recursive):
if file['file_type'] == 'f':
files.append(File(
path=file['path'],
size=file['length']))
return files
def move(self, source: str, destination: str) -> bool:
return len(list(self._client.rename([source], destination))) > 0
def remove(self, path: str) -> bool:
return len(list(self._client.delete([path]))) > 0
def copy(self, source: str, destination: str) -> bool:
# TODO
raise NotImplementedError()
def mkdir(self, path: str) -> bool:
return next(self._client.mkdir([path], create_parent=True))\
.get('result')
def mkdir(hdfs_path, create_parent=False, mode=0755):
"""
paths (list of strings) : Paths to create
create_parent (boolean) : Also create the parent directories
mode (int) : Mode the directory should be created with
Returns:
String mkdir result as json
"""
client = AutoConfigClient()
return list(client.mkdir([hdfs_path], create_parent, mode))
logger.info("Getting user list from /etc/passwd and ldap")
# get a sorted user list from the passwd directory (file+ldap)
user_list = sorted(pwd.getpwall(), key=lambda tup: tup[0])
for user in user_list:
username = user.pw_name
userdir = "/user/" + username
if user.pw_uid <= 500:
continue
if user.pw_uid >= 65534:
continue
if client.test(userdir, exists=True):
logger.debug("User exists " + username)
else:
logger.info("username doesn't exist " + username + "; Creating")
if list(client.mkdir([userdir]))[0]['result']:
logger.info("Created " + userdir)
if list(client.chown([userdir], username))[0]['result']:
logger.info("Chowning userdir to " + username)
else:
logger.warn("Could not chown to user" + username)
if client.test(userdir, exists=True):
logger.info("... created")
else:
logger.warn("Could not create /user/" + username)
logger.info("finished.")