def _import_dir_helper(source_path, target_path, overwrite, exclude_hidden_files):
# Try doing the os.listdir before creating the dir in Databricks.
filenames = os.listdir(source_path)
if exclude_hidden_files:
# for now, just exclude hidden files or directories based on starting '.'
filenames = [f for f in filenames if not f.startswith('.')]
try:
mkdirs(target_path)
except HTTPError as e:
click.echo(e.response.json())
return
for filename in filenames:
cur_src = os.path.join(source_path, filename)
# don't use os.path.join here since it will set \ on Windows
cur_dst = target_path.rstrip('/') + '/' + filename
if os.path.isdir(cur_src):
_import_dir_helper(cur_src, cur_dst, overwrite, exclude_hidden_files)
elif os.path.isfile(cur_src):
ext = WorkspaceLanguage.get_extension(cur_src)
if ext != '':
cur_dst = cur_dst[:-len(ext)]
(language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src)
import_workspace(cur_src, cur_dst, language, file_format, overwrite)
click.echo('{} -> {}'.format(cur_src, cur_dst))
else:
extensions = ', '.join(WorkspaceLanguage.EXTENSIONS)
click.echo(('{} does not have a valid extension of {}. Skip this file and ' +
'continue.').format(cur_src, extensions))
def _import_dir_helper(source_path, target_path, overwrite):
# Try doing the os.listdir before creating the dir in Databricks.
filenames = os.listdir(source_path)
try:
mkdirs(target_path)
except HTTPError as e:
click.echo(e.response.json())
return
for filename in filenames:
cur_src = os.path.join(source_path, filename)
cur_dst = os.path.join(target_path, filename)
if os.path.isdir(cur_src):
_import_dir_helper(cur_src, cur_dst, overwrite)
elif os.path.isfile(cur_src):
ext = WorkspaceLanguage.get_extension(cur_src)
if ext != '':
cur_dst = cur_dst.rstrip(ext)
language = WorkspaceLanguage.to_language(cur_src)
import_workspace(cur_src, cur_dst, language,
WorkspaceFormat.SOURCE, overwrite)
click.echo('{} -> {}'.format(cur_src, cur_dst))
else:
extensions = ', '.join(WorkspaceLanguage.EXTENSIONS)
click.echo((
'{} does not have a valid extension of {}. Skip this file and '
+ 'continue.').format(cur_src, extensions))
def import_workspace_cli(source_path, target_path, language, format, overwrite): # NOQA
"""
Imports a file from local to the Databricks workspace.
The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPTYER, and DBC. Each
format is documented at
https://docs.databricks.com/api/latest/workspace.html#notebookexportformat.
"""
import_workspace(source_path, target_path, language, format, overwrite) # NOQA
def test_import_workspace(tmpdir):
with mock.patch('databricks_cli.workspace.api.get_workspace_client'
) as get_workspace_client:
test_file_path = os.path.join(tmpdir.strpath, 'test')
with open(test_file_path, 'w') as f:
f.write('test')
api.import_workspace(test_file_path,
TEST_WORKSPACE_PATH,
TEST_LANGUAGE,
TEST_FMT,
is_overwrite=False)
import_workspace_mock = get_workspace_client.return_value.import_workspace
assert import_workspace_mock.call_count == 1
assert import_workspace_mock.call_args[0][0] == TEST_WORKSPACE_PATH
assert import_workspace_mock.call_args[0][1] == TEST_FMT
assert import_workspace_mock.call_args[0][2] == TEST_LANGUAGE
assert import_workspace_mock.call_args[0][3] == b64encode('test')
assert import_workspace_mock.call_args[0][4] == False
