def validate_git_fat_in_path() -> None:
"""Validate that git-fat is in the path, asssuming we already know that this
is a git-fat repo.
If the executable is not found, throw a configuration error. We need to do this, as git itself
will not return an error return code if a filter (e.g. git-fat) is not found.
"""
find_exe("git-fat", GIT_FAT_ERRMSG, additional_search_locations=[])
def validate_git_fat_in_path_if_needed(repo_dir: str) -> None:
"""Validate that git-fat is in the path, if this repo is git-fat enabled.
Otherwise, throw a configuration error. We need to do this, as git itself
will not return an error return code if a filter (e.g. git-fat) is not found.
"""
if not is_a_git_fat_repo(repo_dir):
return
find_exe("git-fat", GIT_FAT_ERRMSG, additional_search_locations=[])
def setUp(self):
if os.path.exists(TEMPDIR):
shutil.rmtree(TEMPDIR)
os.mkdir(TEMPDIR)
os.mkdir(WS_DIR)
self.dws = find_exe(
"dws",
"Make sure you have enabled your python virtual environment")
def find_git_lfs_in_path() -> str:
"""Validate that git-lfs is in the path, asssuming we already know that this
is a git-lfs repo.
If the executable is not found, throw a configuration error. We need to do this, as git itself
will not return an error return code if a filter (e.g. git-lfs) is not found.
Retuns the path to the git-lfs executable.
"""
return find_exe("git-lfs", GIT_LFS_ERRMSG, additional_search_locations=[])
def setUp(self):
if exists(TEMPDIR):
shutil.rmtree(TEMPDIR)
os.mkdir(TEMPDIR)
os.mkdir(WS_DIR)
self.dws=find_exe("dws", "Make sure you have enabled your python virtual environment")
self._run_dws(['init', '--hostname', 'test-host',
'--create-resources=code,source-data,intermediate-data,results'],
verbose=False)
shutil.copy(NOTEBOOK, join(CODE_DIR, NOTEBOOK))
def setUp(self):
if isdir(BAD_FAT_DIR):
raise Exception(
"%s exists -- you seem to have run git fat init against the actual dataworkspaces repo!"
% BAD_FAT_DIR)
if os.path.exists(TEMPDIR):
shutil.rmtree(TEMPDIR)
os.mkdir(TEMPDIR)
os.mkdir(WS_DIR)
os.mkdir(FAT_FILES)
os.mkdir(CLONED_WS_PARENT)
self.dws = find_exe(
"dws",
"Make sure you have enabled your python virtual environment")
def ensure_git_lfs_configured_if_needed(repo_dir: str, verbose: bool = False) -> None:
"""If this repo uses git-lfs, then 1) validate that git-lfs is in the path,
and 2) run git-lfs install for the user, if needed.
If the repo uses git-lfs, but we cannot find the executable,
throw a configuration error. We need to do this, as git itself
will not return an error return code if a filter (e.g. git-lfs) is not found.
"""
if not is_a_git_lfs_repo(repo_dir, recursive=True):
return
lfs_exe = find_exe("git-lfs", GIT_LFS_ERRMSG, additional_search_locations=[])
need_to_download = ensure_git_lfs_installed_for_user(lfs_exe, verbose=verbose)
if need_to_download:
# If the user wasn't configured for git-lfs when cloning, we need to
# explicitly download the files.
call_subprocess([lfs_exe, "fetch"], cwd=repo_dir, verbose=verbose)
call_subprocess([lfs_exe, "checkout"], cwd=repo_dir, verbose=verbose)
def setUp(self):
if exists(TEMPDIR):
shutil.rmtree(TEMPDIR)
os.mkdir(TEMPDIR)
os.mkdir(WS_DIR)
self.dws = find_exe(
"dws",
"Make sure you have enabled your python virtual environment")
self._run_dws([
'init', '--hostname', 'test-host',
'--create-resources=code,source-data,intermediate-data,results'
],
verbose=False)
with open(join(WS_DIR, 'source-data/data.csv'), 'w') as f:
f.write('a,b,c\n')
f.write('1,2,3\n')
shutil.copyfile(join(TEST_DIR, 'lineage_step1.py'),
join(CODE_DIR, 'lineage_step1.py'))
shutil.copyfile(join(TEST_DIR, 'lineage_step2.py'),
join(CODE_DIR, 'lineage_step2.py'))
shutil.copyfile(join(TEST_DIR, 'lineage_params_step.py'),
join(CODE_DIR, 'lineage_params_step.py'))
import dataworkspaces
except ImportError:
sys.path.append(os.path.abspath(".."))
from dataworkspaces.utils.subprocess_utils import find_exe
TEMPDIR=os.path.abspath(os.path.expanduser(__file__)).replace('.py', '_data')
WS_DIR=join(TEMPDIR, 'workspace')
CODE_DIR=join(WS_DIR, 'code')
NOTEBOOK='test_jupyter_kit.ipynb'
PYTHONPATH=os.path.abspath("..")
try:
JUPYTER=find_exe('jupyter', 'install Jupyter before running this test')
ERROR = None
except Exception as e:
ERROR = e
JUPYTER=None
try:
import pandas
except ImportError:
pandas = None
try:
import numpy
except ImportError:
numpy = None
@unittest.skipUnless(JUPYTER is not None, "SKIP: No Jupyter install found: %s"%ERROR)
GZIP_EXE = exe
break
assert GZIP_EXE is not None,\
"Could not find gzip. Looked for it at: %s" % ', '.join(GZIP_CANDIDATES)
from dataworkspaces.utils.git_utils import GIT_EXE_PATH
from dataworkspaces.utils.subprocess_utils import find_exe
from dataworkspaces.errors import ConfigurationError
from dataworkspaces.utils.git_lfs_utils import \
_does_attributes_file_reference_lfs, is_a_git_lfs_repo,\
is_git_lfs_installed_for_user
try:
lfs_exe = find_exe(
'git-lfs',
'install git-lfs via instructions at https://git-lfs.github.com')
GIT_LFS_INSTALLED = True
print("git-lfs installed at %s" % lfs_exe)
except:
print("Did not find git-lfs")
GIT_LFS_INSTALLED = False
def make_compressed_file(path, extra=None):
if exists(path + '.gz'):
os.remove(path + '.gz')
with open(path, 'w') as f:
f.write(
"This is a test. This file will be compressed and stored in git-files\n"
)
def rclone_found():
try:
find_exe('rclone', "need to install rclone")
return True
except:
return False