def validate_git_fat_in_path() -> None: """Validate that git-fat is in the path, asssuming we already know that this is a git-fat repo. If the executable is not found, throw a configuration error. We need to do this, as git itself will not return an error return code if a filter (e.g. git-fat) is not found. """ find_exe("git-fat", GIT_FAT_ERRMSG, additional_search_locations=[])
def validate_git_fat_in_path_if_needed(repo_dir: str) -> None: """Validate that git-fat is in the path, if this repo is git-fat enabled. Otherwise, throw a configuration error. We need to do this, as git itself will not return an error return code if a filter (e.g. git-fat) is not found. """ if not is_a_git_fat_repo(repo_dir): return find_exe("git-fat", GIT_FAT_ERRMSG, additional_search_locations=[])
def setUp(self): if os.path.exists(TEMPDIR): shutil.rmtree(TEMPDIR) os.mkdir(TEMPDIR) os.mkdir(WS_DIR) self.dws = find_exe( "dws", "Make sure you have enabled your python virtual environment")
def find_git_lfs_in_path() -> str: """Validate that git-lfs is in the path, asssuming we already know that this is a git-lfs repo. If the executable is not found, throw a configuration error. We need to do this, as git itself will not return an error return code if a filter (e.g. git-lfs) is not found. Retuns the path to the git-lfs executable. """ return find_exe("git-lfs", GIT_LFS_ERRMSG, additional_search_locations=[])
def setUp(self): if exists(TEMPDIR): shutil.rmtree(TEMPDIR) os.mkdir(TEMPDIR) os.mkdir(WS_DIR) self.dws=find_exe("dws", "Make sure you have enabled your python virtual environment") self._run_dws(['init', '--hostname', 'test-host', '--create-resources=code,source-data,intermediate-data,results'], verbose=False) shutil.copy(NOTEBOOK, join(CODE_DIR, NOTEBOOK))
def setUp(self): if isdir(BAD_FAT_DIR): raise Exception( "%s exists -- you seem to have run git fat init against the actual dataworkspaces repo!" % BAD_FAT_DIR) if os.path.exists(TEMPDIR): shutil.rmtree(TEMPDIR) os.mkdir(TEMPDIR) os.mkdir(WS_DIR) os.mkdir(FAT_FILES) os.mkdir(CLONED_WS_PARENT) self.dws = find_exe( "dws", "Make sure you have enabled your python virtual environment")
def ensure_git_lfs_configured_if_needed(repo_dir: str, verbose: bool = False) -> None: """If this repo uses git-lfs, then 1) validate that git-lfs is in the path, and 2) run git-lfs install for the user, if needed. If the repo uses git-lfs, but we cannot find the executable, throw a configuration error. We need to do this, as git itself will not return an error return code if a filter (e.g. git-lfs) is not found. """ if not is_a_git_lfs_repo(repo_dir, recursive=True): return lfs_exe = find_exe("git-lfs", GIT_LFS_ERRMSG, additional_search_locations=[]) need_to_download = ensure_git_lfs_installed_for_user(lfs_exe, verbose=verbose) if need_to_download: # If the user wasn't configured for git-lfs when cloning, we need to # explicitly download the files. call_subprocess([lfs_exe, "fetch"], cwd=repo_dir, verbose=verbose) call_subprocess([lfs_exe, "checkout"], cwd=repo_dir, verbose=verbose)
def setUp(self): if exists(TEMPDIR): shutil.rmtree(TEMPDIR) os.mkdir(TEMPDIR) os.mkdir(WS_DIR) self.dws = find_exe( "dws", "Make sure you have enabled your python virtual environment") self._run_dws([ 'init', '--hostname', 'test-host', '--create-resources=code,source-data,intermediate-data,results' ], verbose=False) with open(join(WS_DIR, 'source-data/data.csv'), 'w') as f: f.write('a,b,c\n') f.write('1,2,3\n') shutil.copyfile(join(TEST_DIR, 'lineage_step1.py'), join(CODE_DIR, 'lineage_step1.py')) shutil.copyfile(join(TEST_DIR, 'lineage_step2.py'), join(CODE_DIR, 'lineage_step2.py')) shutil.copyfile(join(TEST_DIR, 'lineage_params_step.py'), join(CODE_DIR, 'lineage_params_step.py'))
import dataworkspaces except ImportError: sys.path.append(os.path.abspath("..")) from dataworkspaces.utils.subprocess_utils import find_exe TEMPDIR=os.path.abspath(os.path.expanduser(__file__)).replace('.py', '_data') WS_DIR=join(TEMPDIR, 'workspace') CODE_DIR=join(WS_DIR, 'code') NOTEBOOK='test_jupyter_kit.ipynb' PYTHONPATH=os.path.abspath("..") try: JUPYTER=find_exe('jupyter', 'install Jupyter before running this test') ERROR = None except Exception as e: ERROR = e JUPYTER=None try: import pandas except ImportError: pandas = None try: import numpy except ImportError: numpy = None @unittest.skipUnless(JUPYTER is not None, "SKIP: No Jupyter install found: %s"%ERROR)
GZIP_EXE = exe break assert GZIP_EXE is not None,\ "Could not find gzip. Looked for it at: %s" % ', '.join(GZIP_CANDIDATES) from dataworkspaces.utils.git_utils import GIT_EXE_PATH from dataworkspaces.utils.subprocess_utils import find_exe from dataworkspaces.errors import ConfigurationError from dataworkspaces.utils.git_lfs_utils import \ _does_attributes_file_reference_lfs, is_a_git_lfs_repo,\ is_git_lfs_installed_for_user try: lfs_exe = find_exe( 'git-lfs', 'install git-lfs via instructions at https://git-lfs.github.com') GIT_LFS_INSTALLED = True print("git-lfs installed at %s" % lfs_exe) except: print("Did not find git-lfs") GIT_LFS_INSTALLED = False def make_compressed_file(path, extra=None): if exists(path + '.gz'): os.remove(path + '.gz') with open(path, 'w') as f: f.write( "This is a test. This file will be compressed and stored in git-files\n" )
def rclone_found(): try: find_exe('rclone', "need to install rclone") return True except: return False