def test_make_sure_path_exists(self): self.assertTrue(utils.make_sure_path_exists("/usr/")) self.assertTrue(utils.make_sure_path_exists("tests/blah")) self.assertTrue(utils.make_sure_path_exists("tests/trailingslash/")) self.assertFalse(utils.make_sure_path_exists("/this-dir-does-not-exist-and-cant-be-created/")) shutil.rmtree("tests/blah/") shutil.rmtree("tests/trailingslash/")
def render_and_create_dir(dirname, context, output_dir, environment, overwrite_if_exists=False): """Render name of a directory, create the directory, return its path.""" name_tmpl = environment.from_string(dirname) rendered_dirname = name_tmpl.render(**context) dir_to_create = os.path.normpath(os.path.join(output_dir, rendered_dirname)) logger.debug('Rendered dir %s must exist in output_dir %s', dir_to_create, output_dir) output_dir_exists = os.path.exists(dir_to_create) if output_dir_exists: if overwrite_if_exists: logger.debug('Output directory %s already exists, overwriting it', dir_to_create) else: msg = 'Error: "{}" directory already exists'.format(dir_to_create) raise OutputDirExistsException(msg) else: make_sure_path_exists(dir_to_create) return dir_to_create, not output_dir_exists
def test_make_sure_path_exists(self): self.assertTrue(utils.make_sure_path_exists('/usr/')) self.assertTrue(utils.make_sure_path_exists('tests/blah')) self.assertTrue(utils.make_sure_path_exists('tests/trailingslash/')) self.assertFalse( utils.make_sure_path_exists( '/this-dir-does-not-exist-and-cant-be-created/')) shutil.rmtree('tests/blah/') shutil.rmtree('tests/trailingslash/')
def clone(repo_url, checkout=None, clone_to_dir='.', no_input=False): """ Clone a repo to the current directory. :param repo_url: Repo URL of unknown type. :param checkout: The branch, tag or commit ID to checkout after clone. :param clone_to_dir: The directory to clone to. Defaults to the current directory. :param no_input: Suppress all user prompts when calling via API. :returns: str with path to the new directory of the repository. """ # Ensure that clone_to_dir exists clone_to_dir = os.path.expanduser(clone_to_dir) make_sure_path_exists(clone_to_dir) # identify the repo_type vcs, repo_url = identify_repo(repo_url) # check that the appropriate VCS for the repo_type is installed if not vcs.is_installed(): msg = "'{0}' is not installed.".format(vcs.cmd) raise VCSNotInstalled(msg) repo_url = repo_url.rstrip('/') repo_name = os.path.split(repo_url)[1] repo_dir = vcs.get_repo_dir(repo_name, clone_to_dir) logger.debug('repo_dir is {0}'.format(repo_dir)) if os.path.isdir(repo_dir): clone = prompt_and_delete(repo_dir, no_input=no_input) else: clone = True if clone: try: vcs.clone(repo_url, checkout, clone_to_dir, repo_dir) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode('utf-8') # In case of error, print VCS output click.echo( 'Cloning of {} repository {} returned an error:\n{}'.format( vcs.cmd, repo_url, output)) if any(error in output.lower() for error in vcs.not_found_errors): raise RepositoryNotFound( 'The repository {} could not be found, ' 'have you made a typo?'.format(repo_url)) if any(error in output.lower() for error in vcs.branch_errors): raise RepositoryCloneFailed( 'The {} branch of repository {} could not found, ' 'have you made a typo?'.format(checkout, repo_url)) # Raise base subprocess error if SVN error can't be identified raise return repo_dir
def test_make_sure_path_exists(self): self.assertTrue(utils.make_sure_path_exists('/usr/')) self.assertTrue(utils.make_sure_path_exists('tests/blah')) self.assertTrue(utils.make_sure_path_exists('tests/trailingslash/')) self.assertFalse( utils.make_sure_path_exists( '/this-dir-does-not-exist-and-cant-be-created/'.replace("/", os.sep) ) ) shutil.rmtree('tests/blah/') shutil.rmtree('tests/trailingslash/')
def test_make_sure_path_exists(): if sys.platform.startswith('win'): existing_directory = os.path.abspath(os.curdir) uncreatable_directory = 'a*b' else: existing_directory = '/usr/' uncreatable_directory = '/this-doesnt-exist-and-cant-be-created/' assert utils.make_sure_path_exists(existing_directory) assert utils.make_sure_path_exists('tests/blah') assert utils.make_sure_path_exists('tests/trailingslash/') assert not utils.make_sure_path_exists(uncreatable_directory) utils.rmtree('tests/blah/') utils.rmtree('tests/trailingslash/')
def test_make_sure_path_exists(self): if sys.platform.startswith('win'): existing_directory = os.path.abspath(os.curdir) uncreatable_directory = 'a*b' else: existing_directory = '/usr/' uncreatable_directory = '/this-dir-does-not-exist-and-cant-be-created/' self.assertTrue(utils.make_sure_path_exists(existing_directory)) self.assertTrue(utils.make_sure_path_exists('tests/blah')) self.assertTrue(utils.make_sure_path_exists('tests/trailingslash/')) self.assertFalse(utils.make_sure_path_exists(uncreatable_directory)) utils.rmtree('tests/blah/') utils.rmtree('tests/trailingslash/')
def test_make_sure_path_exists(): if sys.platform.startswith("win"): existing_directory = os.path.abspath(os.curdir) uncreatable_directory = "a*b" else: existing_directory = "/usr/" uncreatable_directory = "/this-doesnt-exist-and-cant-be-created/" assert utils.make_sure_path_exists(existing_directory) assert utils.make_sure_path_exists("tests/blah") assert utils.make_sure_path_exists("tests/trailingslash/") assert not utils.make_sure_path_exists(uncreatable_directory) utils.rmtree("tests/blah/") utils.rmtree("tests/trailingslash/")
def fetch_ansible_roles(): roles = [] with open('requirements.ansible_roles', 'r') as rfile: my_roles = rfile.read().rstrip().split(',') if len(my_roles) > 0: roles = my_roles roles_dir = 'ansible/roles' make_sure_path_exists(roles_dir) for role in roles: repo_url = '[email protected]:ephes/ansible_{}.git'.format(role) role_dir = os.path.join(roles_dir, role) if not os.path.exists(role_dir): cloned_dir = clone(repo_url, checkout='master', clone_to_dir=roles_dir) shutil.move(cloned_dir, os.path.join(roles_dir, role))
def dump(replay_dir, template_name, context): # pragma: no cover """dump our replay to disk. [description] :param replay_dir: Where should the replay go ? :type replay_dir: str :param template_name: originating template name :type template_name: str :param context: the context applied to the template :type context: dict """ if not make_sure_path_exists(replay_dir): raise IOError('Unable to create replay dir at {}'.format(replay_dir)) if not isinstance(template_name, str): raise TypeError('Template name is required to be of type str') if not isinstance(context, dict): raise TypeError('Context is required to be of type dict') replay_file = get_file_name(replay_dir, template_name) with open(replay_file, 'w') as outfile: json.dump(context, outfile)
def test_make_sure_path_exists(tmp_path): """Verify correct True/False response from `utils.make_sure_path_exists`. Should return True if directory exist or created. Should return False if impossible to create directory (for example protected) """ existing_directory = tmp_path directory_to_create = Path(tmp_path, "not_yet_created") assert utils.make_sure_path_exists(existing_directory) assert utils.make_sure_path_exists(directory_to_create) # Ensure by base system methods. assert existing_directory.is_dir() assert existing_directory.exists() assert directory_to_create.is_dir() assert directory_to_create.exists()
def test_make_sure_path_exists(): """Verify correct True/False response from `utils.make_sure_path_exists`. Should return True if directory exist or created. Should return False if impossible to create directory (for example protected) """ if sys.platform.startswith('win'): existing_directory = os.path.abspath(os.curdir) uncreatable_directory = 'a*b' else: existing_directory = '/usr/' uncreatable_directory = '/this-doesnt-exist-and-cant-be-created/' assert utils.make_sure_path_exists(existing_directory) assert utils.make_sure_path_exists('tests/blah') assert utils.make_sure_path_exists('tests/trailingslash/') assert not utils.make_sure_path_exists(uncreatable_directory) utils.rmtree('tests/blah/') utils.rmtree('tests/trailingslash/')
def add_ssh_pubkeys(): pubkey_paths = [] with open('requirements.ansible_ssh_pubkeys', 'r') as rfile: my_pubkey_paths = rfile.read().rstrip().split(',') if len(my_pubkey_paths) > 0: for my_path in my_pubkey_paths: my_path = os.path.expanduser(my_path) my_path = os.path.abspath(my_path) pubkey_paths.append(my_path) pubkeys = [] for pubkey_path in pubkey_paths: with open(pubkey_path) as pfile: pubkey = pfile.read().rstrip() if len(pubkey) > 0: pubkeys.append(pubkey) pubkeys_dir = 'ansible/roles/common/files/public_keys' make_sure_path_exists(pubkeys_dir) with open(os.path.join(pubkeys_dir, 'pubkeys'), 'w') as pubkeyfile: pubkeyfile.write('\n'.join(pubkeys))
def test_make_sure_path_exists_correctly_handle_os_error(mocker): """Verify correct True/False response from `utils.make_sure_path_exists`. Should return True if directory exist or created. Should return False if impossible to create directory (for example protected) """ def raiser(*args, **kwargs): raise OSError() mocker.patch("os.makedirs", raiser) uncreatable_directory = Path('protected_path') assert not utils.make_sure_path_exists(uncreatable_directory)
def dump(replay_dir, template_name, context): """Write json data to file.""" if not make_sure_path_exists(replay_dir): raise IOError('Unable to create replay dir at {}'.format(replay_dir)) if not isinstance(template_name, str): raise TypeError('Template name is required to be of type str') if not isinstance(context, dict): raise TypeError('Context is required to be of type dict') replay_file = get_file_name(replay_dir, template_name) with open(replay_file, 'w') as outfile: json.dump(context, outfile, indent=2)
def apply_cookiecutter(self): """ If target_dir doesn't have a cookiecutter template, then apply one. """ exists = make_sure_path_exists(self.target_dir) if exists: print(self.target_dir) cookiecutter( "https://github.com/drivendata/cookiecutter-data-science", no_input=True, output_dir=self.target_dir, ) else: print("target_dir does not exist") return
def dump(replay_dir, template_name, context): """Write json data to file.""" if not make_sure_path_exists(replay_dir): raise IOError("Unable to create replay dir at {}".format(replay_dir)) if not isinstance(template_name, six.string_types): raise TypeError("Template name is required to be of type str") if not isinstance(context, dict): raise TypeError("Context is required to be of type dict") if "cookiecutter" not in context: raise ValueError("Context is required to contain a cookiecutter key") replay_file = get_file_name(replay_dir, template_name) with open(replay_file, "w") as outfile: json.dump(context, outfile)
def unzip(zip_uri, is_url, clone_to_dir='.', no_input=False, refresh=False, password=None): """Download and unpack a zipfile at a given URI. This will download the zipfile to the cookiecutter repository, and unpack into a temporary directory. :param zip_uri: The URI for the zipfile. :param is_url: Is the zip URI a URL or a file? :param clone_to_dir: The cookiecutter repository directory to put the archive into. :param no_input: Suppress any prompts :param refresh: If true, overwrites cached cookiecutter without prompting user :param password: The password to use when unpacking the repository. """ # Ensure that clone_to_dir exists clone_to_dir = os.path.expanduser(clone_to_dir) make_sure_path_exists(clone_to_dir) if is_url: # Build the name of the cached zipfile, # and prompt to delete if it already exists. identifier = zip_uri.rsplit('/', 1)[1] zip_path = os.path.join(clone_to_dir, identifier) if os.path.exists(zip_path): download = prompt_and_delete(zip_path, no_input=no_input, refresh=refresh) else: download = True if download: # (Re) download the zipfile r = requests.get(zip_uri, stream=True) with open(zip_path, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) else: # Just use the local zipfile as-is. zip_path = os.path.abspath(zip_uri) # Now unpack the repository. The zipfile will be unpacked # into a temporary directory try: zip_file = ZipFile(zip_path) if len(zip_file.namelist()) == 0: raise InvalidZipRepository( 'Zip repository {} is empty'.format(zip_uri)) # The first record in the zipfile should be the directory entry for # the archive. If it isn't a directory, there's a problem. first_filename = zip_file.namelist()[0] if not first_filename.endswith('/'): raise InvalidZipRepository('Zip repository {} does not include ' 'a top-level directory'.format(zip_uri)) # Construct the final target directory project_name = first_filename[:-1] unzip_base = tempfile.mkdtemp() unzip_path = os.path.join(unzip_base, project_name) # Extract the zip file into the temporary directory try: zip_file.extractall(path=unzip_base) except RuntimeError: # File is password protected; try to get a password from the # environment; if that doesn't work, ask the user. if password is not None: try: zip_file.extractall(path=unzip_base, pwd=password.encode('utf-8')) except RuntimeError: raise InvalidZipRepository( 'Invalid password provided for protected repository') elif no_input: raise InvalidZipRepository( 'Unable to unlock password protected repository') else: retry = 0 while retry is not None: try: password = read_repo_password('Repo password') zip_file.extractall(path=unzip_base, pwd=password.encode('utf-8')) retry = None except RuntimeError: retry += 1 if retry == 3: raise InvalidZipRepository( 'Invalid password provided for protected repository' ) except BadZipFile: raise InvalidZipRepository( 'Zip repository {} is not a valid zip archive:'.format(zip_uri)) return unzip_path
def clone(repo_url, checkout=None, clone_to_dir='.', no_input=False): """Clone a repo to the current directory. :param repo_url: Repo URL of unknown type. :param checkout: The branch, tag or commit ID to checkout after clone. :param clone_to_dir: The directory to clone to. Defaults to the current directory. :param no_input: Suppress all user prompts when calling via API. :returns: str with path to the new directory of the repository. """ # Ensure that clone_to_dir exists clone_to_dir = os.path.expanduser(clone_to_dir) make_sure_path_exists(clone_to_dir) # identify the repo_type repo_type, repo_url = identify_repo(repo_url) # check that the appropriate VCS for the repo_type is installed if not is_vcs_installed(repo_type): msg = "'{0}' is not installed.".format(repo_type) raise VCSNotInstalled(msg) repo_url = repo_url.rstrip('/') repo_name = os.path.split(repo_url)[1] if repo_type == 'git': repo_name = repo_name.split(':')[-1].rsplit('.git')[0] repo_dir = os.path.normpath(os.path.join(clone_to_dir, repo_name)) elif repo_type == 'hg': repo_dir = os.path.normpath(os.path.join(clone_to_dir, repo_name)) logger.debug('repo_dir is {0}'.format(repo_dir)) if os.path.isdir(repo_dir): clone = prompt_and_delete(repo_dir, no_input=no_input) else: clone = True if clone: try: subprocess.check_output( # nosec [repo_type, 'clone', repo_url], cwd=clone_to_dir, stderr=subprocess.STDOUT, ) if checkout is not None: subprocess.check_output( # nosec [repo_type, 'checkout', checkout], cwd=repo_dir, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode('utf-8') if 'not found' in output.lower(): raise RepositoryNotFound( 'The repository {} could not be found, ' 'have you made a typo?'.format(repo_url)) if any(error in output for error in BRANCH_ERRORS): raise RepositoryCloneFailed( 'The {} branch of repository {} could not found, ' 'have you made a typo?'.format(checkout, repo_url)) logger.error('git clone failed with error: %s', output) raise return repo_dir
def clone(repo_url, checkout=None, clone_to_dir=".", no_input=False): """Clone a repo to the current directory. :param repo_url: Repo URL of unknown type. :param checkout: The branch, tag or commit ID to checkout after clone. :param clone_to_dir: The directory to clone to. Defaults to the current directory. :param no_input: Suppress all user prompts when calling via API. """ # Ensure that clone_to_dir exists clone_to_dir = os.path.expanduser(clone_to_dir) make_sure_path_exists(clone_to_dir) # identify the repo_type repo_type, repo_url = identify_repo(repo_url) # check that the appropriate VCS for the repo_type is installed if not is_vcs_installed(repo_type): msg = "'{0}' is not installed.".format(repo_type) raise VCSNotInstalled(msg) repo_url = repo_url.rstrip("/") tail = os.path.split(repo_url)[1] if repo_type == "git": repo_dir = os.path.normpath( os.path.join(clone_to_dir, tail.rsplit(".git")[0])) elif repo_type == "hg": repo_dir = os.path.normpath(os.path.join(clone_to_dir, tail)) logger.debug("repo_dir is {0}".format(repo_dir)) if os.path.isdir(repo_dir): clone = prompt_and_delete(repo_dir, no_input=no_input) else: clone = True if clone: try: subprocess.check_output( [repo_type, "clone", repo_url], cwd=clone_to_dir, stderr=subprocess.STDOUT, ) if checkout is not None: subprocess.check_output( [repo_type, "checkout", checkout], cwd=repo_dir, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): raise RepositoryNotFound( "The repository {} could not be found, " "have you made a typo?".format(repo_url)) if any(error in output for error in BRANCH_ERRORS): raise RepositoryCloneFailed( "The {} branch of repository {} could not found, " "have you made a typo?".format(checkout, repo_url)) raise return repo_dir