def _import_dir_helper(source_path, target_path, overwrite, exclude_hidden_files):
# Try doing the os.listdir before creating the dir in Databricks.
filenames = os.listdir(source_path)
if exclude_hidden_files:
# for now, just exclude hidden files or directories based on starting '.'
filenames = [f for f in filenames if not f.startswith('.')]
try:
mkdirs(target_path)
except HTTPError as e:
click.echo(e.response.json())
return
for filename in filenames:
cur_src = os.path.join(source_path, filename)
# don't use os.path.join here since it will set \ on Windows
cur_dst = target_path.rstrip('/') + '/' + filename
if os.path.isdir(cur_src):
_import_dir_helper(cur_src, cur_dst, overwrite, exclude_hidden_files)
elif os.path.isfile(cur_src):
ext = WorkspaceLanguage.get_extension(cur_src)
if ext != '':
cur_dst = cur_dst[:-len(ext)]
(language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src)
import_workspace(cur_src, cur_dst, language, file_format, overwrite)
click.echo('{} -> {}'.format(cur_src, cur_dst))
else:
extensions = ', '.join(WorkspaceLanguage.EXTENSIONS)
click.echo(('{} does not have a valid extension of {}. Skip this file and ' +
'continue.').format(cur_src, extensions))
def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files,
headers=None):
# pylint: disable=too-many-locals
filenames = os.listdir(source_path)
if exclude_hidden_files:
# for now, just exclude hidden files or directories based on starting '.'
filenames = [f for f in filenames if not f.startswith('.')]
try:
self.mkdirs(target_path, headers=headers)
except HTTPError as e:
click.echo(e.response.json())
return
for filename in filenames:
cur_src = os.path.join(source_path, filename)
# don't use os.path.join here since it will set \ on Windows
cur_dst = target_path.rstrip('/') + '/' + filename
if os.path.isdir(cur_src):
self.import_workspace_dir(cur_src, cur_dst, overwrite, exclude_hidden_files,
headers=headers)
elif os.path.isfile(cur_src):
ext = WorkspaceLanguage.get_extension(cur_src)
if ext != '':
cur_dst = cur_dst[:-len(ext)]
(language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src)
self.import_workspace(cur_src, cur_dst, language, file_format, overwrite,
headers=headers)
click.echo('{} -> {}'.format(cur_src, cur_dst))
else:
extensions = ', '.join(WorkspaceLanguage.EXTENSIONS)
click.echo(('{} does not have a valid extension of {}. Skip this file and ' +
'continue.').format(cur_src, extensions))
def _import_dir_helper(source_path, target_path, overwrite):
# Try doing the os.listdir before creating the dir in Databricks.
filenames = os.listdir(source_path)
try:
mkdirs(target_path)
except HTTPError as e:
click.echo(e.response.json())
return
for filename in filenames:
cur_src = os.path.join(source_path, filename)
cur_dst = os.path.join(target_path, filename)
if os.path.isdir(cur_src):
_import_dir_helper(cur_src, cur_dst, overwrite)
elif os.path.isfile(cur_src):
ext = WorkspaceLanguage.get_extension(cur_src)
if ext != '':
cur_dst = cur_dst.rstrip(ext)
language = WorkspaceLanguage.to_language(cur_src)
import_workspace(cur_src, cur_dst, language,
WorkspaceFormat.SOURCE, overwrite)
click.echo('{} -> {}'.format(cur_src, cur_dst))
else:
extensions = ', '.join(WorkspaceLanguage.EXTENSIONS)
click.echo((
'{} does not have a valid extension of {}. Skip this file and '
+ 'continue.').format(cur_src, extensions))
