def test_make_sure_path_exists(self):
self.assertTrue(utils.make_sure_path_exists("/usr/"))
self.assertTrue(utils.make_sure_path_exists("tests/blah"))
self.assertTrue(utils.make_sure_path_exists("tests/trailingslash/"))
self.assertFalse(utils.make_sure_path_exists("/this-dir-does-not-exist-and-cant-be-created/"))
shutil.rmtree("tests/blah/")
shutil.rmtree("tests/trailingslash/")
def render_and_create_dir(dirname,
context,
output_dir,
environment,
overwrite_if_exists=False):
"""Render name of a directory, create the directory, return its path."""
name_tmpl = environment.from_string(dirname)
rendered_dirname = name_tmpl.render(**context)
dir_to_create = os.path.normpath(os.path.join(output_dir,
rendered_dirname))
logger.debug('Rendered dir %s must exist in output_dir %s', dir_to_create,
output_dir)
output_dir_exists = os.path.exists(dir_to_create)
if output_dir_exists:
if overwrite_if_exists:
logger.debug('Output directory %s already exists, overwriting it',
dir_to_create)
else:
msg = 'Error: "{}" directory already exists'.format(dir_to_create)
raise OutputDirExistsException(msg)
else:
make_sure_path_exists(dir_to_create)
return dir_to_create, not output_dir_exists
def test_make_sure_path_exists(self):
self.assertTrue(utils.make_sure_path_exists('/usr/'))
self.assertTrue(utils.make_sure_path_exists('tests/blah'))
self.assertTrue(utils.make_sure_path_exists('tests/trailingslash/'))
self.assertFalse(
utils.make_sure_path_exists(
'/this-dir-does-not-exist-and-cant-be-created/'))
shutil.rmtree('tests/blah/')
shutil.rmtree('tests/trailingslash/')
def clone(repo_url, checkout=None, clone_to_dir='.', no_input=False):
"""
Clone a repo to the current directory.
:param repo_url: Repo URL of unknown type.
:param checkout: The branch, tag or commit ID to checkout after clone.
:param clone_to_dir: The directory to clone to.
Defaults to the current directory.
:param no_input: Suppress all user prompts when calling via API.
:returns: str with path to the new directory of the repository.
"""
# Ensure that clone_to_dir exists
clone_to_dir = os.path.expanduser(clone_to_dir)
make_sure_path_exists(clone_to_dir)
# identify the repo_type
vcs, repo_url = identify_repo(repo_url)
# check that the appropriate VCS for the repo_type is installed
if not vcs.is_installed():
msg = "'{0}' is not installed.".format(vcs.cmd)
raise VCSNotInstalled(msg)
repo_url = repo_url.rstrip('/')
repo_name = os.path.split(repo_url)[1]
repo_dir = vcs.get_repo_dir(repo_name, clone_to_dir)
logger.debug('repo_dir is {0}'.format(repo_dir))
if os.path.isdir(repo_dir):
clone = prompt_and_delete(repo_dir, no_input=no_input)
else:
clone = True
if clone:
try:
vcs.clone(repo_url, checkout, clone_to_dir, repo_dir)
except subprocess.CalledProcessError as clone_error:
output = clone_error.output.decode('utf-8')
# In case of error, print VCS output
click.echo(
'Cloning of {} repository {} returned an error:\n{}'.format(
vcs.cmd, repo_url, output))
if any(error in output.lower() for error in vcs.not_found_errors):
raise RepositoryNotFound(
'The repository {} could not be found, '
'have you made a typo?'.format(repo_url))
if any(error in output.lower() for error in vcs.branch_errors):
raise RepositoryCloneFailed(
'The {} branch of repository {} could not found, '
'have you made a typo?'.format(checkout, repo_url))
# Raise base subprocess error if SVN error can't be identified
raise
return repo_dir
def test_make_sure_path_exists(self):
self.assertTrue(utils.make_sure_path_exists('/usr/'))
self.assertTrue(utils.make_sure_path_exists('tests/blah'))
self.assertTrue(utils.make_sure_path_exists('tests/trailingslash/'))
self.assertFalse(
utils.make_sure_path_exists(
'/this-dir-does-not-exist-and-cant-be-created/'.replace("/", os.sep)
)
)
shutil.rmtree('tests/blah/')
shutil.rmtree('tests/trailingslash/')
def test_make_sure_path_exists():
if sys.platform.startswith('win'):
existing_directory = os.path.abspath(os.curdir)
uncreatable_directory = 'a*b'
else:
existing_directory = '/usr/'
uncreatable_directory = '/this-doesnt-exist-and-cant-be-created/'
assert utils.make_sure_path_exists(existing_directory)
assert utils.make_sure_path_exists('tests/blah')
assert utils.make_sure_path_exists('tests/trailingslash/')
assert not utils.make_sure_path_exists(uncreatable_directory)
utils.rmtree('tests/blah/')
utils.rmtree('tests/trailingslash/')
def test_make_sure_path_exists(self):
if sys.platform.startswith('win'):
existing_directory = os.path.abspath(os.curdir)
uncreatable_directory = 'a*b'
else:
existing_directory = '/usr/'
uncreatable_directory = '/this-dir-does-not-exist-and-cant-be-created/'
self.assertTrue(utils.make_sure_path_exists(existing_directory))
self.assertTrue(utils.make_sure_path_exists('tests/blah'))
self.assertTrue(utils.make_sure_path_exists('tests/trailingslash/'))
self.assertFalse(utils.make_sure_path_exists(uncreatable_directory))
utils.rmtree('tests/blah/')
utils.rmtree('tests/trailingslash/')
def test_make_sure_path_exists():
if sys.platform.startswith("win"):
existing_directory = os.path.abspath(os.curdir)
uncreatable_directory = "a*b"
else:
existing_directory = "/usr/"
uncreatable_directory = "/this-doesnt-exist-and-cant-be-created/"
assert utils.make_sure_path_exists(existing_directory)
assert utils.make_sure_path_exists("tests/blah")
assert utils.make_sure_path_exists("tests/trailingslash/")
assert not utils.make_sure_path_exists(uncreatable_directory)
utils.rmtree("tests/blah/")
utils.rmtree("tests/trailingslash/")
def fetch_ansible_roles():
roles = []
with open('requirements.ansible_roles', 'r') as rfile:
my_roles = rfile.read().rstrip().split(',')
if len(my_roles) > 0:
roles = my_roles
roles_dir = 'ansible/roles'
make_sure_path_exists(roles_dir)
for role in roles:
repo_url = '[email protected]:ephes/ansible_{}.git'.format(role)
role_dir = os.path.join(roles_dir, role)
if not os.path.exists(role_dir):
cloned_dir = clone(repo_url, checkout='master', clone_to_dir=roles_dir)
shutil.move(cloned_dir, os.path.join(roles_dir, role))
def dump(replay_dir, template_name, context): # pragma: no cover
"""dump our replay to disk.
[description]
:param replay_dir: Where should the replay go ?
:type replay_dir: str
:param template_name: originating template name
:type template_name: str
:param context: the context applied to the template
:type context: dict
"""
if not make_sure_path_exists(replay_dir):
raise IOError('Unable to create replay dir at {}'.format(replay_dir))
if not isinstance(template_name, str):
raise TypeError('Template name is required to be of type str')
if not isinstance(context, dict):
raise TypeError('Context is required to be of type dict')
replay_file = get_file_name(replay_dir, template_name)
with open(replay_file, 'w') as outfile:
json.dump(context, outfile)
def test_make_sure_path_exists(tmp_path):
"""Verify correct True/False response from `utils.make_sure_path_exists`.
Should return True if directory exist or created.
Should return False if impossible to create directory (for example protected)
"""
existing_directory = tmp_path
directory_to_create = Path(tmp_path, "not_yet_created")
assert utils.make_sure_path_exists(existing_directory)
assert utils.make_sure_path_exists(directory_to_create)
# Ensure by base system methods.
assert existing_directory.is_dir()
assert existing_directory.exists()
assert directory_to_create.is_dir()
assert directory_to_create.exists()
def test_make_sure_path_exists():
"""Verify correct True/False response from `utils.make_sure_path_exists`.
Should return True if directory exist or created.
Should return False if impossible to create directory (for example protected)
"""
if sys.platform.startswith('win'):
existing_directory = os.path.abspath(os.curdir)
uncreatable_directory = 'a*b'
else:
existing_directory = '/usr/'
uncreatable_directory = '/this-doesnt-exist-and-cant-be-created/'
assert utils.make_sure_path_exists(existing_directory)
assert utils.make_sure_path_exists('tests/blah')
assert utils.make_sure_path_exists('tests/trailingslash/')
assert not utils.make_sure_path_exists(uncreatable_directory)
utils.rmtree('tests/blah/')
utils.rmtree('tests/trailingslash/')
def add_ssh_pubkeys():
pubkey_paths = []
with open('requirements.ansible_ssh_pubkeys', 'r') as rfile:
my_pubkey_paths = rfile.read().rstrip().split(',')
if len(my_pubkey_paths) > 0:
for my_path in my_pubkey_paths:
my_path = os.path.expanduser(my_path)
my_path = os.path.abspath(my_path)
pubkey_paths.append(my_path)
pubkeys = []
for pubkey_path in pubkey_paths:
with open(pubkey_path) as pfile:
pubkey = pfile.read().rstrip()
if len(pubkey) > 0:
pubkeys.append(pubkey)
pubkeys_dir = 'ansible/roles/common/files/public_keys'
make_sure_path_exists(pubkeys_dir)
with open(os.path.join(pubkeys_dir, 'pubkeys'), 'w') as pubkeyfile:
pubkeyfile.write('\n'.join(pubkeys))
def test_make_sure_path_exists_correctly_handle_os_error(mocker):
"""Verify correct True/False response from `utils.make_sure_path_exists`.
Should return True if directory exist or created.
Should return False if impossible to create directory (for example protected)
"""
def raiser(*args, **kwargs):
raise OSError()
mocker.patch("os.makedirs", raiser)
uncreatable_directory = Path('protected_path')
assert not utils.make_sure_path_exists(uncreatable_directory)
def dump(replay_dir, template_name, context):
"""Write json data to file."""
if not make_sure_path_exists(replay_dir):
raise IOError('Unable to create replay dir at {}'.format(replay_dir))
if not isinstance(template_name, str):
raise TypeError('Template name is required to be of type str')
if not isinstance(context, dict):
raise TypeError('Context is required to be of type dict')
replay_file = get_file_name(replay_dir, template_name)
with open(replay_file, 'w') as outfile:
json.dump(context, outfile, indent=2)
def apply_cookiecutter(self):
"""
If target_dir doesn't have a cookiecutter template, then apply one.
"""
exists = make_sure_path_exists(self.target_dir)
if exists:
print(self.target_dir)
cookiecutter(
"https://github.com/drivendata/cookiecutter-data-science",
no_input=True,
output_dir=self.target_dir,
)
else:
print("target_dir does not exist")
return
def dump(replay_dir, template_name, context):
"""Write json data to file."""
if not make_sure_path_exists(replay_dir):
raise IOError("Unable to create replay dir at {}".format(replay_dir))
if not isinstance(template_name, six.string_types):
raise TypeError("Template name is required to be of type str")
if not isinstance(context, dict):
raise TypeError("Context is required to be of type dict")
if "cookiecutter" not in context:
raise ValueError("Context is required to contain a cookiecutter key")
replay_file = get_file_name(replay_dir, template_name)
with open(replay_file, "w") as outfile:
json.dump(context, outfile)
def unzip(zip_uri,
is_url,
clone_to_dir='.',
no_input=False,
refresh=False,
password=None):
"""Download and unpack a zipfile at a given URI.
This will download the zipfile to the cookiecutter repository,
and unpack into a temporary directory.
:param zip_uri: The URI for the zipfile.
:param is_url: Is the zip URI a URL or a file?
:param clone_to_dir: The cookiecutter repository directory
to put the archive into.
:param no_input: Suppress any prompts
:param refresh: If true, overwrites cached cookiecutter without prompting user
:param password: The password to use when unpacking the repository.
"""
# Ensure that clone_to_dir exists
clone_to_dir = os.path.expanduser(clone_to_dir)
make_sure_path_exists(clone_to_dir)
if is_url:
# Build the name of the cached zipfile,
# and prompt to delete if it already exists.
identifier = zip_uri.rsplit('/', 1)[1]
zip_path = os.path.join(clone_to_dir, identifier)
if os.path.exists(zip_path):
download = prompt_and_delete(zip_path,
no_input=no_input,
refresh=refresh)
else:
download = True
if download:
# (Re) download the zipfile
r = requests.get(zip_uri, stream=True)
with open(zip_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
else:
# Just use the local zipfile as-is.
zip_path = os.path.abspath(zip_uri)
# Now unpack the repository. The zipfile will be unpacked
# into a temporary directory
try:
zip_file = ZipFile(zip_path)
if len(zip_file.namelist()) == 0:
raise InvalidZipRepository(
'Zip repository {} is empty'.format(zip_uri))
# The first record in the zipfile should be the directory entry for
# the archive. If it isn't a directory, there's a problem.
first_filename = zip_file.namelist()[0]
if not first_filename.endswith('/'):
raise InvalidZipRepository('Zip repository {} does not include '
'a top-level directory'.format(zip_uri))
# Construct the final target directory
project_name = first_filename[:-1]
unzip_base = tempfile.mkdtemp()
unzip_path = os.path.join(unzip_base, project_name)
# Extract the zip file into the temporary directory
try:
zip_file.extractall(path=unzip_base)
except RuntimeError:
# File is password protected; try to get a password from the
# environment; if that doesn't work, ask the user.
if password is not None:
try:
zip_file.extractall(path=unzip_base,
pwd=password.encode('utf-8'))
except RuntimeError:
raise InvalidZipRepository(
'Invalid password provided for protected repository')
elif no_input:
raise InvalidZipRepository(
'Unable to unlock password protected repository')
else:
retry = 0
while retry is not None:
try:
password = read_repo_password('Repo password')
zip_file.extractall(path=unzip_base,
pwd=password.encode('utf-8'))
retry = None
except RuntimeError:
retry += 1
if retry == 3:
raise InvalidZipRepository(
'Invalid password provided for protected repository'
)
except BadZipFile:
raise InvalidZipRepository(
'Zip repository {} is not a valid zip archive:'.format(zip_uri))
return unzip_path
def clone(repo_url, checkout=None, clone_to_dir='.', no_input=False):
"""Clone a repo to the current directory.
:param repo_url: Repo URL of unknown type.
:param checkout: The branch, tag or commit ID to checkout after clone.
:param clone_to_dir: The directory to clone to.
Defaults to the current directory.
:param no_input: Suppress all user prompts when calling via API.
:returns: str with path to the new directory of the repository.
"""
# Ensure that clone_to_dir exists
clone_to_dir = os.path.expanduser(clone_to_dir)
make_sure_path_exists(clone_to_dir)
# identify the repo_type
repo_type, repo_url = identify_repo(repo_url)
# check that the appropriate VCS for the repo_type is installed
if not is_vcs_installed(repo_type):
msg = "'{0}' is not installed.".format(repo_type)
raise VCSNotInstalled(msg)
repo_url = repo_url.rstrip('/')
repo_name = os.path.split(repo_url)[1]
if repo_type == 'git':
repo_name = repo_name.split(':')[-1].rsplit('.git')[0]
repo_dir = os.path.normpath(os.path.join(clone_to_dir, repo_name))
elif repo_type == 'hg':
repo_dir = os.path.normpath(os.path.join(clone_to_dir, repo_name))
logger.debug('repo_dir is {0}'.format(repo_dir))
if os.path.isdir(repo_dir):
clone = prompt_and_delete(repo_dir, no_input=no_input)
else:
clone = True
if clone:
try:
subprocess.check_output( # nosec
[repo_type, 'clone', repo_url],
cwd=clone_to_dir,
stderr=subprocess.STDOUT,
)
if checkout is not None:
subprocess.check_output( # nosec
[repo_type, 'checkout', checkout],
cwd=repo_dir,
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as clone_error:
output = clone_error.output.decode('utf-8')
if 'not found' in output.lower():
raise RepositoryNotFound(
'The repository {} could not be found, '
'have you made a typo?'.format(repo_url))
if any(error in output for error in BRANCH_ERRORS):
raise RepositoryCloneFailed(
'The {} branch of repository {} could not found, '
'have you made a typo?'.format(checkout, repo_url))
logger.error('git clone failed with error: %s', output)
raise
return repo_dir
def clone(repo_url, checkout=None, clone_to_dir=".", no_input=False):
"""Clone a repo to the current directory.
:param repo_url: Repo URL of unknown type.
:param checkout: The branch, tag or commit ID to checkout after clone.
:param clone_to_dir: The directory to clone to.
Defaults to the current directory.
:param no_input: Suppress all user prompts when calling via API.
"""
# Ensure that clone_to_dir exists
clone_to_dir = os.path.expanduser(clone_to_dir)
make_sure_path_exists(clone_to_dir)
# identify the repo_type
repo_type, repo_url = identify_repo(repo_url)
# check that the appropriate VCS for the repo_type is installed
if not is_vcs_installed(repo_type):
msg = "'{0}' is not installed.".format(repo_type)
raise VCSNotInstalled(msg)
repo_url = repo_url.rstrip("/")
tail = os.path.split(repo_url)[1]
if repo_type == "git":
repo_dir = os.path.normpath(
os.path.join(clone_to_dir,
tail.rsplit(".git")[0]))
elif repo_type == "hg":
repo_dir = os.path.normpath(os.path.join(clone_to_dir, tail))
logger.debug("repo_dir is {0}".format(repo_dir))
if os.path.isdir(repo_dir):
clone = prompt_and_delete(repo_dir, no_input=no_input)
else:
clone = True
if clone:
try:
subprocess.check_output(
[repo_type, "clone", repo_url],
cwd=clone_to_dir,
stderr=subprocess.STDOUT,
)
if checkout is not None:
subprocess.check_output(
[repo_type, "checkout", checkout],
cwd=repo_dir,
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as clone_error:
output = clone_error.output.decode("utf-8")
if "not found" in output.lower():
raise RepositoryNotFound(
"The repository {} could not be found, "
"have you made a typo?".format(repo_url))
if any(error in output for error in BRANCH_ERRORS):
raise RepositoryCloneFailed(
"The {} branch of repository {} could not found, "
"have you made a typo?".format(checkout, repo_url))
raise
return repo_dir