def _import_dir_helper(source_path, target_path, overwrite, exclude_hidden_files): # Try doing the os.listdir before creating the dir in Databricks. filenames = os.listdir(source_path) if exclude_hidden_files: # for now, just exclude hidden files or directories based on starting '.' filenames = [f for f in filenames if not f.startswith('.')] try: mkdirs(target_path) except HTTPError as e: click.echo(e.response.json()) return for filename in filenames: cur_src = os.path.join(source_path, filename) # don't use os.path.join here since it will set \ on Windows cur_dst = target_path.rstrip('/') + '/' + filename if os.path.isdir(cur_src): _import_dir_helper(cur_src, cur_dst, overwrite, exclude_hidden_files) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) if ext != '': cur_dst = cur_dst[:-len(ext)] (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) import_workspace(cur_src, cur_dst, language, file_format, overwrite) click.echo('{} -> {}'.format(cur_src, cur_dst)) else: extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) click.echo(('{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions))
def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, headers=None): # pylint: disable=too-many-locals filenames = os.listdir(source_path) if exclude_hidden_files: # for now, just exclude hidden files or directories based on starting '.' filenames = [f for f in filenames if not f.startswith('.')] try: self.mkdirs(target_path, headers=headers) except HTTPError as e: click.echo(e.response.json()) return for filename in filenames: cur_src = os.path.join(source_path, filename) # don't use os.path.join here since it will set \ on Windows cur_dst = target_path.rstrip('/') + '/' + filename if os.path.isdir(cur_src): self.import_workspace_dir(cur_src, cur_dst, overwrite, exclude_hidden_files, headers=headers) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) if ext != '': cur_dst = cur_dst[:-len(ext)] (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) self.import_workspace(cur_src, cur_dst, language, file_format, overwrite, headers=headers) click.echo('{} -> {}'.format(cur_src, cur_dst)) else: extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) click.echo(('{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions))
def _import_dir_helper(source_path, target_path, overwrite): # Try doing the os.listdir before creating the dir in Databricks. filenames = os.listdir(source_path) try: mkdirs(target_path) except HTTPError as e: click.echo(e.response.json()) return for filename in filenames: cur_src = os.path.join(source_path, filename) cur_dst = os.path.join(target_path, filename) if os.path.isdir(cur_src): _import_dir_helper(cur_src, cur_dst, overwrite) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) if ext != '': cur_dst = cur_dst.rstrip(ext) language = WorkspaceLanguage.to_language(cur_src) import_workspace(cur_src, cur_dst, language, WorkspaceFormat.SOURCE, overwrite) click.echo('{} -> {}'.format(cur_src, cur_dst)) else: extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) click.echo(( '{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions))