def test_import_export_recursive(self, local_dir):
path = local_dir.strpath
os.chdir(path)
invoke_cli_runner(cli.import_dir_cli, ['-o', '.', WORKSPACE_TEST_PATH])
assert_workspace_file_exists(
os.path.join(WORKSPACE_TEST_PATH, strip_suffix(SCALA_FILE)))
for f in [SCALA_FILE, SQL_FILE, PYTHON_FILE, R_FILE]:
remote_path = os.path.join(
WORKSPACE_TEST_PATH,
os.path.join(LOCAL_TEST_DIR, strip_suffix(f)))
assert_workspace_file_exists(remote_path)
# Copy the data back to `temp-dir`.
local_temp_dir = os.path.join(path, LOCAL_TEMP_DIR)
invoke_cli_runner(cli.export_dir_cli,
[WORKSPACE_TEST_PATH, local_temp_dir])
assert_local_file_contains(os.path.join(local_temp_dir, SCALA_FILE),
SCALA_CONTENTS)
for f, content in [(SCALA_FILE, SCALA_CONTENTS),
(SQL_FILE, SQL_CONTENTS),
(PYTHON_FILE, PYTHON_CONTENTS),
(R_FILE, R_CONTENTS)]:
local_path = os.path.join(local_temp_dir,
os.path.join(LOCAL_TEST_DIR, f))
assert_local_file_contains(local_path, content)
def test_cp_from_remote(self, local_dir):
path = local_dir.strpath
invoke_cli_runner(
cli.cp_cli, [os.path.join(path, LOCAL_TEST_FILE), DBFS_TEST_PATH])
temp_file_path = os.path.join(path, LOCAL_TEMP_FILE)
invoke_cli_runner(
cli.cp_cli,
[os.path.join(DBFS_TEST_PATH, LOCAL_TEST_FILE), temp_file_path])
assert_local_file_content(temp_file_path, TEST_FILE_CONTENTS)
def test_import_export(self, local_dir):
path = local_dir.strpath
invoke_cli_runner(cli.import_workspace_cli, [
os.path.join(path, SCALA_FILE),
os.path.join(WORKSPACE_TEST_PATH, SCALA_FILE), "-l", "scala"
])
assert_workspace_file_exists(
os.path.join(WORKSPACE_TEST_PATH, SCALA_FILE))
invoke_cli_runner(cli.export_workspace_cli, [
os.path.join(WORKSPACE_TEST_PATH, SCALA_FILE),
os.path.join(path, LOCAL_TEMP_FILE)
])
assert_local_file_contains(os.path.join(path, LOCAL_TEMP_FILE),
SCALA_CONTENTS)
def test_cp_recursive(self, local_dir):
path = local_dir.strpath
os.chdir(path)
invoke_cli_runner(cli.cp_cli, ['-r', '.', DBFS_TEST_PATH])
assert_dbfs_file_exists(DbfsPath(DBFS_TEST_PATH).join(LOCAL_TEST_FILE))
assert_dbfs_file_exists(DbfsPath(DBFS_TEST_PATH).join(LOCAL_TEST_DIR))
assert_dbfs_file_exists(
DbfsPath(DBFS_TEST_PATH).join(LOCAL_TEST_FILE_IN_DIR))
# Copy the data back to `temp-dir`.
local_temp_dir = os.path.join(path, LOCAL_TEMP_DIR)
invoke_cli_runner(cli.cp_cli, ['-r', DBFS_TEST_PATH, local_temp_dir])
assert_local_file_content(
os.path.join(local_temp_dir, LOCAL_TEST_FILE), TEST_FILE_CONTENTS)
assert_local_file_content(
os.path.join(local_temp_dir, LOCAL_TEST_FILE_IN_DIR),
TEST_FILE_CONTENTS)
def assert_workspace_file_exists(workspace_path):
dirname, basename = os.path.split(workspace_path)
res = invoke_cli_runner(cli.ls_cli, [dirname])
assert basename in res.output.split("\n")
def workspace_dir():
invoke_cli_runner(cli.mkdirs_cli, [WORKSPACE_TEST_PATH])
yield
invoke_cli_runner(cli.delete_cli, ['-r', WORKSPACE_TEST_PATH])
def test_cp_from_local(self, local_dir):
path = local_dir.strpath
invoke_cli_runner(
cli.cp_cli, [os.path.join(path, LOCAL_TEST_FILE), DBFS_TEST_PATH])
assert_dbfs_file_exists(DbfsPath(DBFS_TEST_PATH).join(LOCAL_TEST_FILE))
def assert_dbfs_file_exists(dbfs_path):
basename = dbfs_path.basename
dirname = dbfs_path.absolute_path[:-len(basename)]
res = invoke_cli_runner(cli.ls_cli, [dirname])
files = res.output.split('\n')
assert basename in files
def dbfs_dir():
invoke_cli_runner(cli.mkdirs_cli, [DBFS_TEST_PATH])
yield
invoke_cli_runner(cli.rm_cli, ['-r', DBFS_TEST_PATH])