def list_files(dbfs_path):
dbfs_api = get_dbfs_client()
list_response = dbfs_api.list(dbfs_path.absolute_path)
if 'files' in list_response:
return [FileInfo.from_json(f) for f in list_response['files']]
else:
return []
def put_file(src_path, dbfs_path, overwrite):
dbfs_api = get_dbfs_client()
handle = dbfs_api.create(dbfs_path.absolute_path, overwrite)['handle']
with open(src_path, 'rb') as local_file:
while True:
contents = local_file.read(BUFFER_SIZE_BYTES)
if len(contents) == 0:
break
dbfs_api.add_block(handle, b64encode(contents))
dbfs_api.close(handle)
def get_file(dbfs_path, dst_path, overwrite):
if os.path.exists(dst_path) and not overwrite:
raise LocalFileExistsException()
dbfs_api = get_dbfs_client()
file_info = get_status(dbfs_path)
if file_info.is_dir:
error_and_quit(('The dbfs file {} is a directory.').format(repr(dbfs_path)))
length = file_info.file_size
offset = 0
with open(dst_path, 'wb') as local_file:
while offset < length:
response = dbfs_api.read(dbfs_path.absolute_path, offset, BUFFER_SIZE_BYTES)
bytes_read = response['bytes_read']
data = response['data']
offset += bytes_read
local_file.write(b64decode(data))
def get_status(dbfs_path):
dbfs_api = get_dbfs_client()
json = dbfs_api.get_status(dbfs_path.absolute_path)
return FileInfo.from_json(json)
def move(dbfs_src, dbfs_dst):
dbfs_api = get_dbfs_client()
dbfs_api.move(dbfs_src.absolute_path, dbfs_dst.absolute_path)
def mkdirs(dbfs_path):
dbfs_api = get_dbfs_client()
dbfs_api.mkdirs(dbfs_path.absolute_path)
def delete(dbfs_path, recursive):
dbfs_api = get_dbfs_client()
dbfs_api.delete(dbfs_path.absolute_path, recursive=recursive)