def list_files(dbfs_path): dbfs_api = get_dbfs_client() list_response = dbfs_api.list(dbfs_path.absolute_path) if 'files' in list_response: return [FileInfo.from_json(f) for f in list_response['files']] else: return []
def put_file(src_path, dbfs_path, overwrite): dbfs_api = get_dbfs_client() handle = dbfs_api.create(dbfs_path.absolute_path, overwrite)['handle'] with open(src_path, 'rb') as local_file: while True: contents = local_file.read(BUFFER_SIZE_BYTES) if len(contents) == 0: break dbfs_api.add_block(handle, b64encode(contents)) dbfs_api.close(handle)
def get_file(dbfs_path, dst_path, overwrite): if os.path.exists(dst_path) and not overwrite: raise LocalFileExistsException() dbfs_api = get_dbfs_client() file_info = get_status(dbfs_path) if file_info.is_dir: error_and_quit(('The dbfs file {} is a directory.').format(repr(dbfs_path))) length = file_info.file_size offset = 0 with open(dst_path, 'wb') as local_file: while offset < length: response = dbfs_api.read(dbfs_path.absolute_path, offset, BUFFER_SIZE_BYTES) bytes_read = response['bytes_read'] data = response['data'] offset += bytes_read local_file.write(b64decode(data))
def get_status(dbfs_path): dbfs_api = get_dbfs_client() json = dbfs_api.get_status(dbfs_path.absolute_path) return FileInfo.from_json(json)
def move(dbfs_src, dbfs_dst): dbfs_api = get_dbfs_client() dbfs_api.move(dbfs_src.absolute_path, dbfs_dst.absolute_path)
def mkdirs(dbfs_path): dbfs_api = get_dbfs_client() dbfs_api.mkdirs(dbfs_path.absolute_path)
def delete(dbfs_path, recursive): dbfs_api = get_dbfs_client() dbfs_api.delete(dbfs_path.absolute_path, recursive=recursive)