def _import_dir_helper(source_path, target_path, overwrite, exclude_hidden_files): # Try doing the os.listdir before creating the dir in Databricks. filenames = os.listdir(source_path) if exclude_hidden_files: # for now, just exclude hidden files or directories based on starting '.' filenames = [f for f in filenames if not f.startswith('.')] try: mkdirs(target_path) except HTTPError as e: click.echo(e.response.json()) return for filename in filenames: cur_src = os.path.join(source_path, filename) # don't use os.path.join here since it will set \ on Windows cur_dst = target_path.rstrip('/') + '/' + filename if os.path.isdir(cur_src): _import_dir_helper(cur_src, cur_dst, overwrite, exclude_hidden_files) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) if ext != '': cur_dst = cur_dst[:-len(ext)] (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) import_workspace(cur_src, cur_dst, language, file_format, overwrite) click.echo('{} -> {}'.format(cur_src, cur_dst)) else: extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) click.echo(('{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions))
def _import_dir_helper(source_path, target_path, overwrite): # Try doing the os.listdir before creating the dir in Databricks. filenames = os.listdir(source_path) try: mkdirs(target_path) except HTTPError as e: click.echo(e.response.json()) return for filename in filenames: cur_src = os.path.join(source_path, filename) cur_dst = os.path.join(target_path, filename) if os.path.isdir(cur_src): _import_dir_helper(cur_src, cur_dst, overwrite) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) if ext != '': cur_dst = cur_dst.rstrip(ext) language = WorkspaceLanguage.to_language(cur_src) import_workspace(cur_src, cur_dst, language, WorkspaceFormat.SOURCE, overwrite) click.echo('{} -> {}'.format(cur_src, cur_dst)) else: extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) click.echo(( '{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions))
def import_workspace_cli(source_path, target_path, language, format, overwrite): # NOQA """ Imports a file from local to the Databricks workspace. The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPTYER, and DBC. Each format is documented at https://docs.databricks.com/api/latest/workspace.html#notebookexportformat. """ import_workspace(source_path, target_path, language, format, overwrite) # NOQA
def test_import_workspace(tmpdir): with mock.patch('databricks_cli.workspace.api.get_workspace_client' ) as get_workspace_client: test_file_path = os.path.join(tmpdir.strpath, 'test') with open(test_file_path, 'w') as f: f.write('test') api.import_workspace(test_file_path, TEST_WORKSPACE_PATH, TEST_LANGUAGE, TEST_FMT, is_overwrite=False) import_workspace_mock = get_workspace_client.return_value.import_workspace assert import_workspace_mock.call_count == 1 assert import_workspace_mock.call_args[0][0] == TEST_WORKSPACE_PATH assert import_workspace_mock.call_args[0][1] == TEST_FMT assert import_workspace_mock.call_args[0][2] == TEST_LANGUAGE assert import_workspace_mock.call_args[0][3] == b64encode('test') assert import_workspace_mock.call_args[0][4] == False