def get_repository_info(recipe_path): """This tries to get information about where a recipe came from. This is different from the source - you can have a recipe in svn that gets source via git.""" try: if exists(join(recipe_path, ".git")): origin = check_output_env(["git", "config", "--get", "remote.origin.url"], cwd=recipe_path) rev = check_output_env(["git", "rev-parse", "HEAD"], cwd=recipe_path) return "Origin {}, commit {}".format(origin, rev) elif isdir(join(recipe_path, ".hg")): origin = check_output_env(["hg", "paths", "default"], cwd=recipe_path) rev = check_output_env(["hg", "id"], cwd=recipe_path).split()[0] return "Origin {}, commit {}".format(origin, rev) elif isdir(join(recipe_path, ".svn")): info = check_output_env(["svn", "info"], cwd=recipe_path) server = re.search("Repository Root: (.*)$", info, flags=re.M).group(1) revision = re.search("Revision: (.*)$", info, flags=re.M).group(1) return "{}, Revision {}".format(server, revision) else: return "{}, last modified {}".format(recipe_path, time.ctime(os.path.getmtime( join(recipe_path, "meta.yaml")))) except CalledProcessError: log.debug("Failed to checkout source in " + recipe_path) return "{}, last modified {}".format(recipe_path, time.ctime(os.path.getmtime( join(recipe_path, "meta.yaml"))))
def get_git_info(repo, config): """ Given a repo to a git repo, return a dictionary of: GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH GIT_BUILD_STR from the output of git describe. :return: """ d = {} log = utils.get_logger(__name__) if config.verbose: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL log.setLevel(logging.ERROR) # grab information from describe env = os.environ.copy() env['GIT_DIR'] = repo keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] try: output = utils.check_output_env( ["git", "describe", "--tags", "--long", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') parts = output.rsplit('-', 2) if len(parts) == 3: d.update(dict(zip(keys, parts))) # get the _full_ hash of the current HEAD output = utils.check_output_env(["git", "rev-parse", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') d['GIT_FULL_HASH'] = output # set up the build string if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d: d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"], d["GIT_DESCRIBE_HASH"]) # issues on Windows with the next line of the command prompt being recorded here. assert not any("\n" in value for value in d.values()) except subprocess.CalledProcessError as error: log.warn( "Error obtaining git information in get_git_info. Error was: ") log.warn(str(error)) return d
def get_git_info(repo, debug): """ Given a repo to a git repo, return a dictionary of: GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH GIT_BUILD_STR from the output of git describe. :return: """ d = {} log = utils.get_logger(__name__) if debug: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL # grab information from describe env = os.environ.copy() env['GIT_DIR'] = repo keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] try: output = utils.check_output_env(["git", "describe", "--tags", "--long", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') parts = output.rsplit('-', 2) if len(parts) == 3: d.update(dict(zip(keys, parts))) except subprocess.CalledProcessError: log.debug("Failed to obtain git tag information. Are you using annotated tags?") try: # get the _full_ hash of the current HEAD output = utils.check_output_env(["git", "rev-parse", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') d['GIT_FULL_HASH'] = output except subprocess.CalledProcessError as error: log.debug("Error obtaining git commit information. Error was: ") log.debug(str(error)) # set up the build string if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d: d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"], d["GIT_DESCRIBE_HASH"]) # issues on Windows with the next line of the command prompt being recorded here. assert not any("\n" in value for value in d.values()) return d
def describe_root(cwd=None): if not cwd: cwd = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) tag = check_output_env(["git", "describe", "--abbrev=0"], cwd=cwd).rstrip() if PY3: tag = tag.decode("utf-8") return tag
def test_subprocess_stats_call(testing_workdir): stats = {} utils.check_call_env(['hostname'], stats=stats, cwd=testing_workdir) assert stats stats = {} out = utils.check_output_env(['hostname'], stats=stats, cwd=testing_workdir) assert out assert stats with pytest.raises(subprocess.CalledProcessError): utils.check_call_env(['bash', '-c', 'exit 1'], cwd=testing_workdir)
def test_subprocess_stats_call(): stats = {} utils.check_call_env(['ls'], stats=stats) assert stats stats = {} out = utils.check_output_env(['ls'], stats=stats) assert out assert stats with pytest.raises(subprocess.CalledProcessError): utils.check_call_env(['bash', '-c', 'exit 1'])
def git_info(src_dir, verbose=True, fo=None): ''' Print info about a Git repo. ''' assert isdir(src_dir) git = external.find_executable('git') if not git: log = get_logger(__name__) log.warn( "git not installed in root environment. Skipping recording of git info." ) return if verbose: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL # Ensure to explicitly set GIT_DIR as some Linux machines will not # properly execute without it. env = os.environ.copy() env['GIT_DIR'] = join(src_dir, '.git') env = {str(key): str(value) for key, value in env.items()} for cmd, check_error in [('git log -n1', True), ('git describe --tags --dirty', False), ('git status', True)]: try: stdout = check_output_env(cmd.split(), stderr=stderr, cwd=src_dir, env=env) except CalledProcessError as e: if check_error: raise Exception("git error: %s" % str(e)) encoding = locale.getpreferredencoding() if not fo: encoding = sys.stdout.encoding encoding = encoding or 'utf-8' if hasattr(stdout, 'decode'): stdout = stdout.decode(encoding, 'ignore') if fo: fo.write(u'==> %s <==\n' % cmd) if verbose: fo.write(stdout + u'\n') else: if verbose: print(u'==> %s <==\n' % cmd) safe_print_unicode(stdout + u'\n')
def git_info(src_dir, verbose=True, fo=None): ''' Print info about a Git repo. ''' assert isdir(src_dir) git = external.find_executable('git') if not git: log = get_logger(__name__) log.warn("git not installed in root environment. Skipping recording of git info.") return if verbose: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL # Ensure to explicitly set GIT_DIR as some Linux machines will not # properly execute without it. env = os.environ.copy() env['GIT_DIR'] = join(src_dir, '.git') env = {str(key): str(value) for key, value in env.items()} for cmd, check_error in [ ('git log -n1', True), ('git describe --tags --dirty', False), ('git status', True)]: try: stdout = check_output_env(cmd.split(), stderr=stderr, cwd=src_dir, env=env) except CalledProcessError as e: if check_error: raise Exception("git error: %s" % str(e)) encoding = locale.getpreferredencoding() if not fo: encoding = sys.stdout.encoding encoding = encoding or 'utf-8' if hasattr(stdout, 'decode'): stdout = stdout.decode(encoding, 'ignore') if fo: fo.write(u'==> %s <==\n' % cmd) if verbose: fo.write(stdout + u'\n') else: if verbose: print(u'==> %s <==\n' % cmd) safe_print_unicode(stdout + u'\n')
def get_hg_build_info(repo): env = os.environ.copy() env['HG_DIR'] = repo env = {str(key): str(value) for key, value in env.items()} d = {} cmd = ["hg", "log", "--template", "{rev}|{node|short}|{latesttag}|{latesttagdistance}|{branch}", "--rev", "."] output = utils.check_output_env(cmd, env=env, cwd=os.path.dirname(repo)) output = output.decode('utf-8') rev, short_id, tag, distance, branch = output.split('|') if tag != 'null': d['HG_LATEST_TAG'] = tag if branch == "": branch = 'default' d['HG_BRANCH'] = branch d['HG_NUM_ID'] = rev d['HG_LATEST_TAG_DISTANCE'] = distance d['HG_SHORT_ID'] = short_id d['HG_BUILD_STR'] = '{}_{}'.format(d['HG_NUM_ID'], d['HG_SHORT_ID']) return d
def git_mirror_checkout_recursive(git, mirror_dir, checkout_dir, git_url, config, git_ref=None, git_depth=-1, is_top_level=True): """ Mirror (and checkout) a Git repository recursively. It's not possible to use `git submodule` on a bare repository, so the checkout must be done before we know which submodules there are. Worse, submodules can be identified by using either absolute URLs or relative paths. If relative paths are used those need to be relocated upon mirroring, but you could end up with `../../../../blah` and in that case conda-build could be tricked into writing to the root of the drive and overwriting the system folders unless steps are taken to prevent that. """ if config.verbose: stdout = None stderr = None else: FNULL = open(os.devnull, 'w') stdout = FNULL stderr = FNULL if not mirror_dir.startswith(config.git_cache + os.sep): sys.exit("Error: Attempting to mirror to %s which is outside of GIT_CACHE %s" % (mirror_dir, config.git_cache)) # This is necessary for Cygwin git and m2-git, although it is fixed in newer MSYS2. git_mirror_dir = convert_path_for_cygwin_or_msys2(git, mirror_dir) git_checkout_dir = convert_path_for_cygwin_or_msys2(git, checkout_dir) if not isdir(os.path.dirname(mirror_dir)): os.makedirs(os.path.dirname(mirror_dir)) if isdir(mirror_dir): if git_ref != 'HEAD': check_call_env([git, 'fetch'], cwd=mirror_dir, stdout=stdout, stderr=stderr) else: # Unlike 'git clone', fetch doesn't automatically update the cache's HEAD, # So here we explicitly store the remote HEAD in the cache's local refs/heads, # and then explicitly set the cache's HEAD. # This is important when the git repo is a local path like "git_url: ../", # but the user is working with a branch other than 'master' without # explicitly providing git_rev. check_call_env([git, 'fetch', 'origin', '+HEAD:_conda_cache_origin_head'], cwd=mirror_dir, stdout=stdout, stderr=stderr) check_call_env([git, 'symbolic-ref', 'HEAD', 'refs/heads/_conda_cache_origin_head'], cwd=mirror_dir, stdout=stdout, stderr=stderr) else: args = [git, 'clone', '--mirror'] if git_depth > 0: args += ['--depth', str(git_depth)] try: check_call_env(args + [git_url, git_mirror_dir], stdout=stdout, stderr=stderr) except CalledProcessError: # on windows, remote URL comes back to us as cygwin or msys format. Python doesn't # know how to normalize it. Need to convert it to a windows path. if sys.platform == 'win32' and git_url.startswith('/'): git_url = convert_unix_path_to_win(git_url) if os.path.exists(git_url): # Local filepaths are allowed, but make sure we normalize them git_url = normpath(git_url) check_call_env(args + [git_url, git_mirror_dir], stdout=stdout, stderr=stderr) assert isdir(mirror_dir) # Now clone from mirror_dir into checkout_dir. check_call_env([git, 'clone', git_mirror_dir, git_checkout_dir], stdout=stdout, stderr=stderr) if is_top_level: checkout = git_ref if git_url.startswith('.'): output = check_output_env([git, "rev-parse", checkout], stdout=stdout, stderr=stderr) checkout = output.decode('utf-8') if config.verbose: print('checkout: %r' % checkout) if checkout: check_call_env([git, 'checkout', checkout], cwd=checkout_dir, stdout=stdout, stderr=stderr) # submodules may have been specified using relative paths. # Those paths are relative to git_url, and will not exist # relative to mirror_dir, unless we do some work to make # it so. try: submodules = check_output_env([git, 'config', '--file', '.gitmodules', '--get-regexp', 'url'], stderr=stdout, cwd=checkout_dir) submodules = submodules.decode('utf-8').splitlines() except CalledProcessError: submodules = [] for submodule in submodules: matches = git_submod_re.match(submodule) if matches and matches.group(2)[0] == '.': submod_name = matches.group(1) submod_rel_path = matches.group(2) submod_url = urljoin(git_url + '/', submod_rel_path) submod_mirror_dir = os.path.normpath( os.path.join(mirror_dir, submod_rel_path)) if config.verbose: print('Relative submodule %s found: url is %s, submod_mirror_dir is %s' % ( submod_name, submod_url, submod_mirror_dir)) with TemporaryDirectory() as temp_checkout_dir: git_mirror_checkout_recursive(git, submod_mirror_dir, temp_checkout_dir, submod_url, config, git_ref, git_depth, False) if is_top_level: # Now that all relative-URL-specified submodules are locally mirrored to # relatively the same place we can go ahead and checkout the submodules. check_call_env([git, 'submodule', 'update', '--init', '--recursive'], cwd=checkout_dir, stdout=stdout, stderr=stderr) git_info(config) if not config.verbose: FNULL.close()
def verify_git_repo(git_dir, git_url, git_commits_since_tag, debug=False, expected_rev='HEAD'): env = os.environ.copy() log = utils.get_logger(__name__) if debug: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL if not expected_rev: return False OK = True env['GIT_DIR'] = git_dir try: # Verify current commit (minus our locally applied patches) matches expected commit current_commit = utils.check_output_env(["git", "log", "-n1", "--format=%H", "HEAD" + "^" * git_commits_since_tag], env=env, stderr=stderr) current_commit = current_commit.decode('utf-8') expected_tag_commit = utils.check_output_env(["git", "log", "-n1", "--format=%H", expected_rev], env=env, stderr=stderr) expected_tag_commit = expected_tag_commit.decode('utf-8') if current_commit != expected_tag_commit: return False # Verify correct remote url. Need to find the git cache directory, # and check the remote from there. cache_details = utils.check_output_env(["git", "remote", "-v"], env=env, stderr=stderr) cache_details = cache_details.decode('utf-8') cache_dir = cache_details.split('\n')[0].split()[1] if not isinstance(cache_dir, str): # On Windows, subprocess env can't handle unicode. cache_dir = cache_dir.encode(sys.getfilesystemencoding() or 'utf-8') try: remote_details = utils.check_output_env(["git", "--git-dir", cache_dir, "remote", "-v"], env=env, stderr=stderr) except subprocess.CalledProcessError: if sys.platform == 'win32' and cache_dir.startswith('/'): cache_dir = utils.convert_unix_path_to_win(cache_dir) remote_details = utils.check_output_env(["git", "--git-dir", cache_dir, "remote", "-v"], env=env, stderr=stderr) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] # on windows, remote URL comes back to us as cygwin or msys format. Python doesn't # know how to normalize it. Need to convert it to a windows path. if sys.platform == 'win32' and remote_url.startswith('/'): remote_url = utils.convert_unix_path_to_win(git_url) if os.path.exists(remote_url): # Local filepaths are allowed, but make sure we normalize them remote_url = normpath(remote_url) # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. if not os.path.isdir(remote_url) and remote_url.lower() != git_url.lower(): log.debug("remote does not match git_url") log.debug("Remote: " + remote_url.lower()) log.debug("git_url: " + git_url.lower()) OK = False except subprocess.CalledProcessError as error: log.debug("Error obtaining git information in verify_git_repo. Error was: ") log.debug(str(error)) OK = False finally: if not debug: FNULL.close() return OK
def verify_git_repo(git_exe, git_dir, git_url, git_commits_since_tag, debug=False, expected_rev='HEAD'): env = os.environ.copy() log = utils.get_logger(__name__) if debug: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL if not expected_rev: return False OK = True env['GIT_DIR'] = git_dir try: # Verify current commit (minus our locally applied patches) matches expected commit current_commit = utils.check_output_env([ git_exe, "log", "-n1", "--format=%H", "HEAD" + "^" * git_commits_since_tag ], env=env, stderr=stderr) current_commit = current_commit.decode('utf-8') expected_tag_commit = utils.check_output_env( [git_exe, "log", "-n1", "--format=%H", expected_rev], env=env, stderr=stderr) expected_tag_commit = expected_tag_commit.decode('utf-8') if current_commit != expected_tag_commit: return False # Verify correct remote url. Need to find the git cache directory, # and check the remote from there. cache_details = utils.check_output_env([git_exe, "remote", "-v"], env=env, stderr=stderr) cache_details = cache_details.decode('utf-8') cache_dir = cache_details.split('\n')[0].split()[1] if not isinstance(cache_dir, str): # On Windows, subprocess env can't handle unicode. cache_dir = cache_dir.encode(sys.getfilesystemencoding() or 'utf-8') try: remote_details = utils.check_output_env( [git_exe, "--git-dir", cache_dir, "remote", "-v"], env=env, stderr=stderr) except subprocess.CalledProcessError: if sys.platform == 'win32' and cache_dir.startswith('/'): cache_dir = utils.convert_unix_path_to_win(cache_dir) remote_details = utils.check_output_env( [git_exe, "--git-dir", cache_dir, "remote", "-v"], env=env, stderr=stderr) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] # on windows, remote URL comes back to us as cygwin or msys format. Python doesn't # know how to normalize it. Need to convert it to a windows path. if sys.platform == 'win32' and remote_url.startswith('/'): remote_url = utils.convert_unix_path_to_win(git_url) if os.path.exists(remote_url): # Local filepaths are allowed, but make sure we normalize them remote_url = normpath(remote_url) # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. if not os.path.isdir( remote_url) and remote_url.lower() != git_url.lower(): log.debug("remote does not match git_url") log.debug("Remote: " + remote_url.lower()) log.debug("git_url: " + git_url.lower()) OK = False except subprocess.CalledProcessError as error: log.debug( "Error obtaining git information in verify_git_repo. Error was: ") log.debug(str(error)) OK = False finally: if not debug: FNULL.close() return OK
def get_git_info(git_exe, repo, debug): """ Given a repo to a git repo, return a dictionary of: GIT_DESCRIBE_TAG GIT_DESCRIBE_TAG_PEP440 GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH GIT_BUILD_STR from the output of git describe. :return: """ d = {} log = utils.get_logger(__name__) if debug: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL # grab information from describe env = os.environ.copy() env['GIT_DIR'] = repo keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] try: output = utils.check_output_env( [git_exe, "describe", "--tags", "--long", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') parts = output.rsplit('-', 2) if len(parts) == 3: d.update(dict(zip(keys, parts))) from conda._vendor.auxlib.packaging import _get_version_from_git_tag d['GIT_DESCRIBE_TAG_PEP440'] = str(_get_version_from_git_tag(output)) except subprocess.CalledProcessError: msg = ("Failed to obtain git tag information.\n" "Consider using annotated tags if you are not already " "as they are more reliable when used with git describe.") log.debug(msg) # If there was no tag reachable from HEAD, the above failed and the short hash is not set. # Try to get the short hash from describing with all refs (not just the tags). if "GIT_DESCRIBE_HASH" not in d: try: output = utils.check_output_env( [git_exe, "describe", "--all", "--long", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') parts = output.rsplit('-', 2) if len(parts) == 3: # Don't save GIT_DESCRIBE_TAG and GIT_DESCRIBE_NUMBER because git (probably) # described a branch. We just want to save the short hash. d['GIT_DESCRIBE_HASH'] = parts[-1] except subprocess.CalledProcessError as error: log.debug("Error obtaining git commit information. Error was: ") log.debug(str(error)) try: # get the _full_ hash of the current HEAD output = utils.check_output_env([git_exe, "rev-parse", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') d['GIT_FULL_HASH'] = output except subprocess.CalledProcessError as error: log.debug("Error obtaining git commit information. Error was: ") log.debug(str(error)) # set up the build string if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d: d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"], d["GIT_DESCRIBE_HASH"]) # issues on Windows with the next line of the command prompt being recorded here. assert not any("\n" in value for value in d.values()) return d
def get_git_info(git_exe, repo, debug): """ Given a repo to a git repo, return a dictionary of: GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH GIT_BUILD_STR from the output of git describe. :return: """ d = {} log = utils.get_logger(__name__) if debug: stderr = None else: FNULL = open(os.devnull, 'w') stderr = FNULL # grab information from describe env = os.environ.copy() env['GIT_DIR'] = repo keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] try: output = utils.check_output_env( [git_exe, "describe", "--tags", "--long", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') parts = output.rsplit('-', 2) if len(parts) == 3: d.update(dict(zip(keys, parts))) except subprocess.CalledProcessError: msg = ("Failed to obtain git tag information.\n" "Consider using annotated tags if you are not already " "as they are more reliable when used with git describe.") log.debug(msg) try: # get the _full_ hash of the current HEAD output = utils.check_output_env([git_exe, "rev-parse", "HEAD"], env=env, cwd=os.path.dirname(repo), stderr=stderr).splitlines()[0] output = output.decode('utf-8') d['GIT_FULL_HASH'] = output except subprocess.CalledProcessError as error: log.debug("Error obtaining git commit information. Error was: ") log.debug(str(error)) # set up the build string if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d: d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"], d["GIT_DESCRIBE_HASH"]) # issues on Windows with the next line of the command prompt being recorded here. assert not any("\n" in value for value in d.values()) return d