def initialize_git_config_map():
"""Initialize the __git_config_map global.
"""
global __git_config_map
# The hooks' configuration is stored on a special branch called
# refs/meta/config, inside a file called project.config. Get
# that file.
(tmp_fd, tmp_file) = mkstemp('tmp-git-hooks-')
try:
cfg_file = tmp_file
try:
git.show('refs/meta/config:project.config', _outfile=tmp_fd)
except CalledProcessError:
# Most likely a project that still uses the repository's
# config file to store the hooks configuration, rather
# that the controlled project.config file.
#
# Handle this situation by doing what we used to do,
# which is get the configuration from the repository's
# config file, after having warned the user about it
# (to expedite the transition).
#
# Note that we cannot use "utils.warn" to do the warning
# in this module, as the "utils" module depends on this
# module. Do the warning by hand.
#
# ??? One small issue is the fact that this warning may get
# displayed multiple times (once per "phase", eg "update",
# then "post-receive"). Given the relatively rare nature
# of this event, we'll just accept it, instead of fancying
# things up.
for l in NO_REFS_META_CONFIG_WARNING.splitlines():
print >> sys.stderr, '*** %s' % l
cfg_file = 'config'
os.close(tmp_fd)
# Get the currently defined config values, all in one go.
# Use "--file <cfg_file>" to make sure that we only parse
# the file we just retrieved. Otherwise, git also parses
# the user's config file.
all_configs = git.config('-l', '--file', cfg_file, _split_lines=True)
finally:
os.unlink(tmp_file)
all_configs_map = dict([config.split('=', 1) for config in all_configs])
# Populate the __git_config_map dictionary...
__git_config_map = {}
for config_name in GIT_CONFIG_OPTS.keys():
# Get the config value from either the all_configs_map
# if defined, or else from the default value.
if config_name in all_configs_map:
config_val = all_configs_map[config_name]
else:
config_val = GIT_CONFIG_OPTS[config_name]['default']
# Finally, save the config value if __git_config_map
__git_config_map[config_name] = config_val
def email_commit(self, commit):
"""Send an email describing the given commit.
PARAMETERS
commit: A CommitInfo object.
"""
if self.ref_namespace in ('refs/heads', 'refs/tags'):
if self.short_ref_name == 'master':
branch = ''
else:
branch = '/%s' % self.short_ref_name
else:
# Unusual namespace for our reference. Use the reference
# name in full to label the branch name.
branch = '(%s)' % self.ref_name
subject = '[%(repo)s%(branch)s] %(subject)s' % {
'repo': self.email_info.project_name,
'branch': branch,
'subject': commit.subject[:SUBJECT_MAX_SUBJECT_CHARS],
}
# Generate the body of the email in two pieces:
# 1. The commit description without the patch;
# 2. The diff stat and patch.
# This allows us to insert our little "Diff:" marker that
# bugtool detects when parsing the email for filing (this
# part is now performed by the Email class). The purpose
# is to prevent bugtool from searching for TNs in the patch
# itself.
#
# For the diff, there is one subtlelty:
# Git commands calls strip on the output, which is usually
# a good thing, but not in the case of the diff output.
# Prevent this from happening by putting an artificial
# character at the start of the format string, and then
# by stripping it from the output.
body = git.log(commit.rev, max_count="1") + '\n'
if git_config('hooks.commit-url') is not None:
url_info = {'rev': commit.rev,
'ref_name': self.ref_name}
body = (git_config('hooks.commit-url') % url_info
+ '\n\n'
+ body)
diff = git.show(commit.rev, p=True, M=True, stat=True,
pretty="format:|")[1:]
filer_cmd = git_config('hooks.file-commit-cmd')
if filer_cmd is not None:
filer_cmd = shlex.split(filer_cmd)
email = Email(self.email_info,
commit.email_to, subject, body, commit.author,
self.ref_name, commit.base_rev_for_display(),
commit.rev, diff, filer_cmd=filer_cmd)
email.enqueue()
def email_commit(self, commit):
"""See AbstractUpdate.email_commit."""
notes = GitNotes(commit.rev)
# Get commit info for the annotated commit
annotated_commit = commit_info_list("-1", notes.annotated_rev)[0]
# Get a description of the annotated commit (a la "git show"),
# except that we do not want the diff.
#
# Also, we have to handle the notes manually, as the commands
# get the notes from the HEAD of the notes/commits branch,
# whereas what we needs is the contents at the commit.rev.
# This makes a difference when a single push updates the notes
# of the same commit multiple times.
annotated_rev_log = git.log(annotated_commit.rev,
no_notes=True,
max_count="1")
notes_contents = (None if notes.contents is None else indent(
notes.contents, ' ' * 4))
# Determine subject tag based on ref name:
# * remove "refs/notes" prefix
# * remove entire tag if remaining component is "commits"
# (case of the default refs/notes/commits ref)
notes_ref = self.ref_name.split('/', 2)[2]
if notes_ref == "commits":
subject_tag = ""
else:
subject_tag = "(%s)" % notes_ref
subject = '[notes%s][%s] %s' % (subject_tag,
self.email_info.project_name,
annotated_commit.subject)
body_template = (DELETED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE
if notes_contents is None else
UPDATED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE)
body = body_template % {
'annotated_rev_log': annotated_rev_log,
'notes_contents': notes_contents,
}
# Git commands calls strip on the output, which is usually
# a good thing, but not in the case of the diff output.
# Prevent this from happening by putting an artificial
# character at the start of the format string, and then
# by stripping it from the output.
diff = git.show(commit.rev, pretty="format:|", p=True)[1:]
email_bcc = git_config('hooks.filer-email')
email = Email(self.email_info,
annotated_commit.email_to(self.ref_name), email_bcc,
subject, body, commit.author, self.ref_name,
commit.base_rev_for_display(), commit.rev, diff)
email.enqueue()
def commit_diff(sha1, git=git):
commit = git.show(sha1)
first_newline = commit.index('\n')
if commit[first_newline+1:].startswith('Merge:'):
return (core.decode(commit) + '\n\n' +
core.decode(diff_helper(commit=sha1,
cached=False,
suppress_header=False)))
else:
return core.decode(commit)
def __get_notes_contents(cls, notes_rev, notes_filename):
"""Return the contents of the notes at notes_rev.
PARAMETERS
notes_rev: The revision of the notes change.
notes_filename: The filename containing the notes.
"""
try:
return git.show('%s:%s' % (notes_rev, notes_filename))
except CalledProcessError:
# The note was probably deleted, so no more notes.
return None
def __get_notes_contents(cls, notes_rev, notes_filename):
"""Return the contents of the notes at notes_rev.
PARAMETERS
notes_rev: The revision of the notes change.
notes_filename: The filename containing the notes.
"""
try:
return git.show('%s:%s' % (notes_rev, notes_filename))
except CalledProcessError:
# The note was probably deleted, so no more notes.
return None
def email_commit(self, commit):
"""See AbstractUpdate.email_commit."""
notes = GitNotes(commit.rev)
# Create a partial CommitInfo object for the commit that
# our note annotates. We create a partial one in order
# to avoid computing some info we do not need...
annotated_commit = CommitInfo(notes.annotated_rev, None, None, None)
# Get a description of the annotated commit (a la "git show"),
# except that we do not want the diff.
#
# Also, we have to handle the notes manually, as the commands
# get the notes from the HEAD of the notes/commits branch,
# whereas what we needs is the contents at the commit.rev.
# This makes a difference when a single push updates the notes
# of the same commit multiple times.
annotated_rev_info = git.log(annotated_commit.rev,
no_notes=True,
max_count="1")
notes_contents = (None if notes.contents is None else indent(
notes.contents, ' ' * 4))
subject = '[%s] notes update for %s' % (self.email_info.project_name,
notes.annotated_rev)
body_template = (DELETED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE
if notes_contents is None else
UPDATED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE)
body = body_template % {
'annotated_rev_info': annotated_rev_info,
'notes_contents': notes_contents,
}
# Git commands calls strip on the output, which is usually
# a good thing, but not in the case of the diff output.
# Prevent this from happening by putting an artificial
# character at the start of the format string, and then
# by stripping it from the output.
diff = git.show(commit.rev, pretty="format:|", p=True)[1:]
email = Email(self.email_info, annotated_commit.email_to, subject,
body, commit.author, self.ref_name,
commit.base_rev_for_display(), commit.rev, diff)
email.enqueue()
def email_commit(self, commit):
"""See AbstractUpdate.email_commit."""
notes = GitNotes(commit.rev)
# Create a partial CommitInfo object for the commit that
# our note annotates. We create a partial one in order
# to avoid computing some info we do not need...
annotated_commit = CommitInfo(notes.annotated_rev, None, None, None)
# Get a description of the annotated commit (a la "git show"),
# except that we do not want the diff.
#
# Also, we have to handle the notes manually, as the commands
# get the notes from the HEAD of the notes/commits branch,
# whereas what we needs is the contents at the commit.rev.
# This makes a difference when a single push updates the notes
# of the same commit multiple times.
annotated_rev_info = git.log(annotated_commit.rev, no_notes=True,
max_count="1")
notes_contents = (None if notes.contents is None
else indent(notes.contents, ' ' * 4))
subject = '[%s] notes update for %s' % (self.email_info.project_name,
notes.annotated_rev)
body_template = (
DELETED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE if notes_contents is None
else UPDATED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE)
body = body_template % {
'annotated_rev_info': annotated_rev_info,
'notes_contents': notes_contents,
}
# Git commands calls strip on the output, which is usually
# a good thing, but not in the case of the diff output.
# Prevent this from happening by putting an artificial
# character at the start of the format string, and then
# by stripping it from the output.
diff = git.show(commit.rev, pretty="format:|", p=True)[1:]
email = Email(self.email_info, annotated_commit.email_to,
subject, body, commit.author, self.ref_name,
commit.base_rev_for_display(), commit.rev, diff)
email.enqueue()
def email_commit(self, commit):
"""Send an email describing the given commit.
PARAMETERS
commit: A CommitInfo object.
"""
if self.ref_namespace in ('refs/heads', 'refs/tags'):
if self.short_ref_name == 'master':
branch = ''
else:
branch = '/%s' % self.short_ref_name
else:
# Unusual namespace for our reference. Use the reference
# name in full to label the branch name.
branch = '(%s)' % self.ref_name
subject = '[%(repo)s%(branch)s] %(subject)s' % {
'repo': self.email_info.project_name,
'branch': branch,
'subject': commit.subject[:SUBJECT_MAX_SUBJECT_CHARS],
}
# Generate the body of the email in two pieces:
# 1. The commit description without the patch;
# 2. The diff stat and patch.
# This allows us to insert our little "Diff:" marker that
# bugtool detects when parsing the email for filing (this
# part is now performed by the Email class). The purpose
# is to prevent bugtool from searching for TNs in the patch
# itself.
#
# For the diff, there is one subtlelty:
# Git commands calls strip on the output, which is usually
# a good thing, but not in the case of the diff output.
# Prevent this from happening by putting an artificial
# character at the start of the format string, and then
# by stripping it from the output.
body = git.log(commit.rev, max_count="1") + '\n'
if git_config('hooks.commit-url') is not None:
url_info = {'rev': commit.rev, 'ref_name': self.ref_name}
body = (git_config('hooks.commit-url') % url_info + '\n\n' + body)
if git_config('hooks.disable-email-diff'):
diff = None
else:
diff = git.show(commit.rev,
p=True,
M=True,
stat=True,
pretty="format:|")[1:]
filer_cmd = git_config('hooks.file-commit-cmd')
if filer_cmd is not None:
filer_cmd = shlex.split(filer_cmd)
email_bcc = git_config('hooks.filer-email')
email = Email(self.email_info,
commit.email_to(self.ref_name),
email_bcc,
subject,
body,
commit.author,
self.ref_name,
commit.base_rev_for_display(),
commit.rev,
diff,
filer_cmd=filer_cmd)
email.enqueue()
def initialize_git_config_map():
"""Initialize the __git_config_map global.
"""
global __git_config_map
# The hooks' configuration is stored in a special reference
# (see CONFIG_REF), inside a file whose name is CONFIG_FILENAME.
# Get that file.
(tmp_fd, tmp_file) = mkstemp('tmp-git-hooks-')
try:
cfg_file = tmp_file
try:
git.show(config_commit + ':' + CONFIG_FILENAME, _outfile=tmp_fd)
except CalledProcessError:
# Either the CONFIG_REF reference does not exist, or
# the config file itself does not exist. Either way,
# it means that the repository has not been properly
# set up for these hooks, which is a fatal error.
raise InvalidUpdate(*CANNOT_FIND_CONFIG_FILE_ERROR.splitlines())
os.close(tmp_fd)
# Get the currently defined config values, all in one go.
# Use "--file <cfg_file>" to make sure that we only parse
# the file we just retrieved. Otherwise, git also parses
# the user's config file.
#
# Also, use the nul character as the separator between each
# entry (-z option) so as to not confuse them with potential
# newlines being used inside the value of an option.
all_configs = git.config('-z', '-l', '--file', cfg_file).split('\x00')
finally:
os.unlink(tmp_file)
all_configs_map = {}
for config in all_configs:
if not config:
# "git config -z" adds a nul character at the end of its output,
# which cause all_configs to end with an empty entry. Just ignore
# those.
continue
config_name, config_val = config.split('\n', 1)
if config_name in GIT_CONFIG_OPTS and \
'type' in GIT_CONFIG_OPTS[config_name] and \
GIT_CONFIG_OPTS[config_name]['type'] == tuple:
# This config is a list of potentially multiple values, and
# therefore multiple entries with the same config name can be
# provided for each value. Just save them in a list.
if config_name not in all_configs_map:
all_configs_map[config_name] = ()
# Also, at least for now, we support coma-separated entries
# for this multiple-value configs. So split each entry as well...
config_val = to_type(config_val, tuple)
all_configs_map[config_name] += config_val
else:
all_configs_map[config_name] = config_val
# Populate the __git_config_map dictionary...
__git_config_map = {}
for config_name in GIT_CONFIG_OPTS.keys():
# Get the config value from either the all_configs_map
# if defined, or else from the default value.
if config_name in all_configs_map:
config_val = all_configs_map[config_name]
else:
config_val = GIT_CONFIG_OPTS[config_name]['default']
# Finally, save the config value if __git_config_map
__git_config_map[config_name] = config_val
def style_check_files(filename_list, commit_rev, project_name):
"""Check a file for style violations if appropriate.
Raise InvalidUpdate if one or more style violations are detected.
PARAMETERS
filename_list: The name of the file to check (an iterable).
commit_rev: The associated commit sha1. This piece of information
helps us find the correct version of the files to be checked,
as well as the .gitattributes files which are used to determine
whether pre-commit-checks should be applied or not.
project_name: The name of the project (same as the attribute
in updates.emails.EmailInfo).
"""
debug(
"style_check_files (commit_rev=%s):\n%s" %
(commit_rev, "\n".join([" - `%s'" % fname
for fname in filename_list])),
level=3,
)
config_file = git_config("hooks.style-checker-config-file")
# Auxilary list of files we need to fetch from the same reference
# for purposes other than checking their contents.
aux_files = []
if config_file is not None and config_file not in filename_list:
if not file_exists(commit_rev, config_file):
info = (STYLE_CHECKER_CONFIG_FILE_MISSING_ERR_MSG % {
"config_filename": config_file,
"commit_rev": commit_rev
}).splitlines()
raise InvalidUpdate(*info)
aux_files.append(config_file)
# Get a copy of all the files and save them in our scratch dir.
# In order to allow us to call the style-checker using
# the full path (from the project's root directory) of
# the files being checked, we re-create the path to those
# filenames, and then copy the files at the same path.
#
# Providing the path as part of the filename argument is useful,
# because it allows the messages printed by the style-checker
# to be unambiguous in the situation where the same project
# has multiple files sharing the same name. More generally,
# it can also be useful to quickly locate a file in the project
# when trying to make the needed corrections outlined by the
# style-checker.
for filename in itertools.chain(filename_list, aux_files):
path_to_filename = "%s/%s" % (utils.scratch_dir,
os.path.dirname(filename))
if not os.path.exists(path_to_filename):
os.makedirs(path_to_filename)
git.show(
"%s:%s" % (commit_rev, filename),
_outfile="%s/%s" % (utils.scratch_dir, filename),
)
# Call the style-checker.
# For testing purposes, provide a back-door allowing the user
# to override the style-checking program to be used. That way,
# the testsuite has a way to control what the program returns,
# and easily test all execution paths without having to maintain
# some sources specifically designed to trigger the various
# error conditions.
style_checker_hook = ThirdPartyHook("hooks.style-checker")
if "GIT_HOOKS_STYLE_CHECKER" in os.environ:
style_checker_hook.hook_exe = os.environ["GIT_HOOKS_STYLE_CHECKER"]
style_checker_hook_args = []
if config_file is not None:
style_checker_hook_args.extend(["--config", config_file])
style_checker_hook_args.append(project_name)
_, p, out = style_checker_hook.call(
hook_input="\n".join(filename_list),
hook_args=style_checker_hook_args,
cwd=utils.scratch_dir,
)
if p.returncode != 0:
info = ["pre-commit check failed for commit: %s" % commit_rev
] + out.splitlines()
raise InvalidUpdate(*info)
# If we reach this point, it means that the style-checker returned
# zero (success). Print any output, it might be a non-fatal warning.
if out:
warn(*out.splitlines())
def __check_gitreview_defaultbranch(self):
"""If .gitreview exists, validate the defaultbranch value.
This is meant to catch the situation where a user creates
a new branch for a repository hosted on gerrit. Those
repositories typically have a .gitreview file at their root,
providing various information, one of them being the default
branch name when sending patches for review. When creating
a new branch, it is very easy (and frequent) for a user to
forget to also update the .gitreview file, causing patch
reviews to be sent with the wrong branch, which later then
causes the patch to be merged (submitted) on the wrong
branch once it is approved.
We try to avoid that situation by reading the contents of
those files at branch creation time, and reporting an error
if it exists and points to a branch name different from ours.
Note that we only do that for the traditional git branches.
We don't worry about the branches in the gerrit-specific
special namespaces, for which a user checking out the branch
and sending a review is unlikely.
"""
GITREVIEW_FILENAME = '.gitreview'
DEFAULTBRANCH_CONFIG_NAME = 'gerrit.defaultbranch'
# Only perform this check for traditional git branches.
# See method description above for the reason why.
if not self.ref_name.startswith('refs/heads/'):
return
if self.search_config_option_list('hooks.no-precommit-check')\
is not None:
# The user explicitly disabled the .gitreview check
# on this branch.
return
# If the file doesn't exist for that branch, then there is
# no problem.
if not file_exists(self.new_rev, GITREVIEW_FILENAME):
return
# Get the contents of the gitreview file, and then get git
# to parse its contents. We process it all into a dictionary
# rather than just query the value of the one config we are
# looking for, for a couple of reasons:
# 1. This allows us to avoid having to git returning with
# and error status when the file does not have the config
# entry we are looking for (git returns error code 1
# in that case);
# 2. If we even want to look at other configurations in
# that file, the code is already in place to do so.
gitreview_contents = git.show('%s:%s' %
(self.new_rev, GITREVIEW_FILENAME))
gitreview_configs = git.config('-z',
'-l',
'--file',
'-',
_input=gitreview_contents).split('\x00')
config_map = {}
for config in gitreview_configs:
if not config:
# "git config -z" adds a nul character at the end of
# its output, which cause gitreview_configs to end with
# an empty entry. Just ignore those.
continue
config_name, config_val = config.split('\n', 1)
config_map[config_name] = config_val
if DEFAULTBRANCH_CONFIG_NAME in config_map and \
config_map[DEFAULTBRANCH_CONFIG_NAME] != self.short_ref_name:
raise InvalidUpdate(
"Incorrect gerrit default branch name in file `%s'." %
GITREVIEW_FILENAME,
"You probably forgot to update your %s file following" %
GITREVIEW_FILENAME, "the creation of this branch.", '',
"Please create a commit which updates the value",
"of %s in the file `%s'" %
(DEFAULTBRANCH_CONFIG_NAME, GITREVIEW_FILENAME),
"and set it to `%s' (instead of `%s')." %
(self.short_ref_name, config_map[DEFAULTBRANCH_CONFIG_NAME]))
def check_file(filename, sha1, commit_rev, project_name):
"""Check a file for style violations if appropriate.
Raise InvalidUpdate if one or more style violations are detected.
PARAMETERS
filename: The name of the file to check.
sha1: The sha1 of the file to be checked.
commit_rev: The associated commit sha1. This piece of information
helps us find the correct version of the .gitattributes files,
in order to determine whether pre-commit-checks should be
applied or not.
project_name: The name of the project (same as the attribute
in updates.emails.EmailInfo).
"""
debug("check_file (filename=`%s', sha1=%s)" % (filename, sha1), level=3)
# Determine whether this file has the no-precommit-check attribute
# set, in which case style-checks should not be performed.
if git_attribute(commit_rev, filename, 'no-precommit-check') == 'set':
debug('no-precommit-check: %s commit_rev=%s'
% (filename, commit_rev))
syslog('Pre-commit checks disabled for %(rev)s on %(repo)s'
' (%(file)s) by repo attribute'
% {'rev': sha1,
'repo': project_name,
'file': filename,
})
return
# Get a copy of the file and save it in our scratch dir.
# In order to allow us to call the style-checker using
# the full path (from the project's root directory) of
# the file being checked, we re-create the path to that
# filename, and then copy the file at that same path.
#
# Providing the path as part of the filename argument is useful,
# because it allows the messages printed by the style-checker
# to be unambiguous in the situation where the same project
# has multiple files sharing the same name. More generally,
# it can also be useful to quickly locate a file in the project
# when trying to make the needed corrections outlined by the
# style-checker.
path_to_filename = "%s/%s" % (utils.scratch_dir,
os.path.dirname(filename))
if not os.path.exists(path_to_filename):
os.makedirs(path_to_filename)
git.show(sha1, _outfile="%s/%s" % (utils.scratch_dir, filename))
# Call the style-checker.
# For testing purposes, provide a back-door allowing the user
# to override the style-checking program to be used. That way,
# the testsuite has a way to control what the program returns,
# and easily test all execution paths without having to maintain
# some sources specifically designed to trigger the various
# error conditions.
if 'GIT_HOOKS_STYLE_CHECKER' in os.environ:
style_checker = os.environ['GIT_HOOKS_STYLE_CHECKER']
else:
style_checker = git_config('hooks.style-checker')
# ??? It appears that cvs_check, the official style-checker,
# requires the SVN path of the file to be checked as the first
# argument. Not sure why, but that does not really apply in
# our context. Use `trunk/<module>/<path>' to work around
# the issue.
style_checker_args = ['trunk/%s/%s' % (project_name, filename),
filename]
try:
# In order to allow the style-checker to be a script, we need to
# run it through a shell. But when we do so, the Popen class no
# longer allows us to pass the arguments as a list. In order to
# avoid problems with spaces or special characters, we quote the
# arguments as needed.
quoted_args = [quote(arg) for arg in style_checker_args]
out = check_output('%s %s' % (style_checker, ' '.join(quoted_args)),
shell=True, cwd=utils.scratch_dir, stderr=STDOUT)
# If we reach this point, it means that the style-checker returned
# zero (success). Print any output, it might be a non-fatal
# warning.
if out:
warn(*out.splitlines())
except subprocess.CalledProcessError, E:
debug(str(E), level=4)
info = (
["pre-commit check failed for file `%s' at commit: %s"
% (filename, commit_rev)]
+ E.output.splitlines())
raise InvalidUpdate(*info)
def git_attribute(commit_rev, filename_list, attr_name):
"""Return filename's attribute value at commit_rev.
PARAMETERS
commit_rev: The commit to use in order to determine the
attribute value. This is important, because more recent
commits may have changed the attribute value through
updates of various .gitattributes files.
filename_list: A list of filenames for which the attribute is
to be determined. The file name should be relative to
the root of the repository.
attr_name: The name of the attribute.
RETURN VALUE
A dictionary, where the key is a the filename (one key for
each file in filename_list), and the value is the file's
attribute value as returned by git (Eg. 'set', 'unset',
'unspecified', etc).
REMARKS
The problem is not as easy as it looks. If we were working
from a full (non-bare) repository, the `git check-attr'
command would give us our answer immediately. But in bare
repositories, the only file read is GIT_DIR/info/attributes.
Originally, we implemented this way: Starting from the directory
where our file is located, find the first .gitattribute file
that specifies an attribute value for our file. Unfortunately,
reading the gitattributes(5) man page more careful, we realized
that this does not implement gitattributes semantics properly
(we don't stop once we found a .gitattributes file with an entry
that matches). Also, this approach turned out to be extremely
slow, and could cause some updates to take minutes to process
for commits where 2-3 thousand files were modified (typical
when updating the copyright year, for instance).
So, instead of trying to re-implement the git-check-attr
command ourselves, what we do now, is create a dummy git
repository inside which we (lazily) reproduce the directory
tree, with their .gitattributes file. And then, from there
call `git check-attr'. And, to help with the performance
aspect, we call it only once requesting the attribute value
for all files all in one go.
"""
# Verify that we have a scratch area we can use for create the fake
# git repository (see REMARKS section above).
assert utils.scratch_dir is not None
# A copy of the environment, but without the GIT_DIR environment
# variable (which gets sets when called by git), pointing to
# the repository to which changes are being pushed. This interferes
# with most git commands when we're trying to work with our fake
# repository. So we use this copy of the environment without
# the GIT_DIR environment variable when needed.
tmp_git_dir_env = dict(os.environ)
tmp_git_dir_env.pop('GIT_DIR', None)
tmp_git_dir = mkdtemp('.git', 'check-attr-', utils.scratch_dir)
git.init(_cwd=tmp_git_dir, _env=tmp_git_dir_env)
# There is one extra complication: We want to also provide support
# for a DEFAULT_ATTRIBUTES_FILE, where the semantics is that,
# if none of the .gitattributes file have an entry matching
# our file, then this file is consulted. Once again, to avoid
# calling `git check-attr' multiple times, what we do instead
# is that we create a the directory tree in a root which is in
# a subdir of tmp_git_dir. That way, we can put the default
# attribute file in the root of tmp_git_dir, and git-check-attr
# will only look at it if checked-in .gitattributes don't define
# the attribute of a given file, thus implementing the "default"
# behavior.
#
# This requires a bit of manipulation, because now, in the fake
# git repository, the files we want to check are conceptually
# inside the subdir. So filenames passed to `git check-attr'
# have to contain that subdir, and the that subdir needs to be
# excised from the command's output.
if isfile(DEFAULT_ATTRIBUTES_FILE):
copy(DEFAULT_ATTRIBUTES_FILE,
os.path.join(tmp_git_dir, ".gitattributes"))
checkout_subdir = 'src'
tmp_checkout_dir = os.path.join(tmp_git_dir, checkout_subdir)
dirs_with_changes = {}
for filename in filename_list:
assert not os.path.isabs(filename)
dir_path = filename
dir_created = False
while dir_path:
dir_path = os.path.dirname(dir_path)
if dir_path in dirs_with_changes:
continue
gitattributes_rel_file = os.path.join(dir_path, '.gitattributes')
if cached_file_exists(commit_rev, gitattributes_rel_file):
if not dir_created:
os.makedirs(os.path.join(tmp_checkout_dir, dir_path))
dir_created = True
git.show("%s:%s" % (commit_rev, gitattributes_rel_file),
_outfile=os.path.join(tmp_checkout_dir,
gitattributes_rel_file))
dirs_with_changes[dir_path] = True
# To avoid having to deal with the parsing of quoted filenames,
# we use the -z option of "git check-attr". What this does is
# that each of the 3 elements of each line is now separated by
# a NUL character. Also, each line now ends with a NUL character
# as well, instead of LF.
#
# To parse the output, we split it at each NUL character.
# This means that the output gets split into a sequence of
# lines which go 3 by 3, with the first line containing
# the filename, the second being the name of the attribute
# being queried, and the third being the attribute's value
# for that file.
check_attr_input = '\x00'.join(['%s/%s' % (checkout_subdir, filename)
for filename in filename_list])
attr_info = git.check_attr('-z', '--stdin', attr_name,
_cwd=tmp_git_dir, _env=tmp_git_dir_env,
_input=check_attr_input).split('\x00')
if len(attr_info) % 3 == 1 and not attr_info[-1]:
# The attribute information for each filename ends with
# a NUL character, so the terminating NUL character in
# the last entry caused the split to add one empty element
# at the end. This is expected, so just remove it.
attr_info.pop()
# As per the above, we should now have a number of lines that's
# a multiple of 3.
assert len(attr_info) % 3 == 0
result = {}
while attr_info:
filename = attr_info.pop(0)
attr_info.pop(0) # Ignore the attribute name...
attr_val = attr_info.pop(0)
assert filename.startswith(checkout_subdir + '/')
filename = filename[len(checkout_subdir) + 1:]
result[filename] = attr_val
return result
def get_standard_commit_email(self, commit):
"""See AbstractUpdate.get_standard_commit_email."""
notes = GitNotes(commit.rev)
# Get commit info for the annotated commit
annotated_commit = commit_info_list("-1", notes.annotated_rev)[0]
# Get a description of the annotated commit (a la "git show"),
# except that we do not want the diff.
#
# Also, we have to handle the notes manually, as the commands
# get the notes from the HEAD of the notes/commits branch,
# whereas what we needs is the contents at the commit.rev.
# This makes a difference when a single push updates the notes
# of the same commit multiple times.
annotated_rev_log = git.log(annotated_commit.rev,
no_notes=True,
max_count="1",
_decode=True)
notes_contents = (None if notes.contents is None else indent(
notes.contents, " " * 4))
# Get the list of references the annotated commit is contained in.
annotated_commit_ref_names = git.for_each_ref(
contains=annotated_commit.rev,
format="%(refname)",
_decode=True,
_split_lines=True,
)
# Strip from that list all the references which are to be ignored
# (typically, those are internal references).
annotated_commit_ref_names = [
ref_name for ref_name in annotated_commit_ref_names
if search_config_option_list("hooks.ignore-refs", ref_name) is None
]
subject_prefix = commit_email_subject_prefix(
project_name=self.email_info.project_name,
ref_names=annotated_commit_ref_names,
)
# Determine subject tag based on ref name:
# * remove "refs/notes" prefix
# * remove entire tag if remaining component is "commits"
# (case of the default refs/notes/commits ref)
notes_ref = self.ref_name.split("/", 2)[2]
if notes_ref == "commits":
subject_tag = ""
else:
subject_tag = "(%s)" % notes_ref
subject = f"[notes{subject_tag}]{subject_prefix} {annotated_commit.subject}"
body_template = (DELETED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE
if notes_contents is None else
UPDATED_NOTES_COMMIT_EMAIL_BODY_TEMPLATE)
body = body_template % {
"annotated_rev_log": annotated_rev_log,
"notes_contents": notes_contents,
}
# Git commands calls strip on the output, which is usually
# a good thing, but not in the case of the diff output.
# Prevent this from happening by putting an artificial
# character at the start of the format string, and then
# by stripping it from the output.
diff = git.show(commit.rev, pretty="format:|", p=True,
_decode=True)[1:]
refs_containing_annotated_commit_section = (
REFS_CONTAINING_ANNOTATED_COMMIT_TEMPLATE.format(
annotated_commit_references="\n".join([
f" {ref_name}"
for ref_name in annotated_commit_ref_names
])))
email_bcc = git_config("hooks.filer-email")
return Email(
self.email_info,
annotated_commit.email_to(self.ref_name),
email_bcc,
subject,
body,
commit.full_author_email,
self.ref_name,
commit.base_rev_for_display(),
commit.rev,
# Place the refs_containing_annotated_commit_section inside
# the "Diff:" section to avoid having that section trigger
# some unexpected filing.
refs_containing_annotated_commit_section + diff,
)
def initialize_git_config_map():
"""Initialize the __git_config_map global.
"""
global __git_config_map
# The hooks' configuration is stored on a special branch called
# refs/meta/config, inside a file called project.config. Get
# that file.
(tmp_fd, tmp_file) = mkstemp('tmp-git-hooks-')
try:
cfg_file = tmp_file
try:
git.show('refs/meta/config:project.config', _outfile=tmp_fd)
except CalledProcessError:
# Most likely a project that still uses the repository's
# config file to store the hooks configuration, rather
# that the controlled project.config file.
#
# Handle this situation by doing what we used to do,
# which is get the configuration from the repository's
# config file, after having warned the user about it
# (to expedite the transition).
#
# Note that we cannot use "utils.warn" to do the warning
# in this module, as the "utils" module depends on this
# module. Do the warning by hand.
#
# ??? One small issue is the fact that this warning may get
# displayed multiple times (once per "phase", eg "update",
# then "post-receive"). Given the relatively rare nature
# of this event, we'll just accept it, instead of fancying
# things up.
for l in NO_REFS_META_CONFIG_WARNING.splitlines():
print >> sys.stderr, '*** %s' % l
cfg_file = 'config'
os.close(tmp_fd)
# Get the currently defined config values, all in one go.
# Use "--file <cfg_file>" to make sure that we only parse
# the file we just retrieved. Otherwise, git also parses
# the user's config file.
#
# Also, use the nul character as the separator between each
# entry (-z option) so as to not confuse them with potential
# newlines being used inside the value of an option.
all_configs = git.config('-z', '-l', '--file', cfg_file).split('\x00')
finally:
os.unlink(tmp_file)
all_configs_map = {}
for config in all_configs:
if not config:
# "git config -z" adds a nul character at the end of its output,
# which cause all_configs to end with an empty entry. Just ignore
# those.
continue
config_name, config_val = config.split('\n', 1)
if config_name in GIT_CONFIG_OPTS and \
'type' in GIT_CONFIG_OPTS[config_name] and \
GIT_CONFIG_OPTS[config_name]['type'] == tuple:
# This config is a list of potentially multiple values, and
# therefore multiple entries with the same config name can be
# provided for each value. Just save them in a list.
if config_name not in all_configs_map:
all_configs_map[config_name] = ()
# Also, at least for now, we support coma-separated entries
# for this multiple-value configs. So split each entry as well...
config_val = to_type(config_val, tuple)
all_configs_map[config_name] += config_val
else:
all_configs_map[config_name] = config_val
# Populate the __git_config_map dictionary...
__git_config_map = {}
for config_name in GIT_CONFIG_OPTS.keys():
# Get the config value from either the all_configs_map
# if defined, or else from the default value.
if config_name in all_configs_map:
config_val = all_configs_map[config_name]
else:
config_val = GIT_CONFIG_OPTS[config_name]['default']
# Finally, save the config value if __git_config_map
__git_config_map[config_name] = config_val
def git_attribute(commit_rev, filename, attr_name):
"""Return filename's attribute value at commit_rev.
PARAMETERS
commit_rev: The commit to use in order to determine the
attribute value. This is important, because more recent
commits may have changed the attribute value through
updates of various .gitattributes files.
filename: The name of the file for which the attribute is
to be determined. The file name should be relative to
the root of the repository.
attr_name: The name of the attribute.
RETURN VALUE
A string containing the attribute value.
REMARKS
The problem is not as easy as it looks. If we were working
from a full (non-bare) repository, the `git check-attr'
command would give us our answer immediately. But in bare
repositories, the only file read is GIT_DIR/info/attributes.
We solve the problem this way. Starting from the directory
where our file is located, find the first .gitattribute file
that specifies an attribute value for our file. If we read
the gitattributes(5) man page correctly, that should yield
the correct answer. Furthermore, if none of the .gitattributes
file yielded a value for this attribute, then try one last
time with GIT_DIR/info/default_attributes (if it exists).
"""
# First, delete any old BARE_REPO_ATTRIBUTES_FILE left from
# the previous push. Otherwise, it causes problems if owned
# by a different user.
if os.path.exists(BARE_REPO_ATTRIBUTES_FILE):
os.remove(BARE_REPO_ATTRIBUTES_FILE)
# Also, if the directory where BARE_REPO_ATTRIBUTES_FILE is stored
# does not exist, create it now. With git version 1.8.1.2, the
# directory is automatically created at repository creation. But
# with git version 1.7.10.4, that's not the case. Easy to handle.
attributes_dir = dirname(BARE_REPO_ATTRIBUTES_FILE)
if not os.path.exists(attributes_dir):
os.makedirs(attributes_dir)
# Depending on how the repository is setup and the user's
# umask, the group-write bit might not be set. Just force
# the permissions to be read-write-execute for both owner
# and group.
os.chmod(attributes_dir, 0775)
(path_to_rel, rel_file_path) = os.path.split(filename)
attr_value = UNSPECIFIED_ATTR
while True:
gitattributes_file = os.path.join(path_to_rel, '.gitattributes')
if cached_file_exists(commit_rev, gitattributes_file):
# Get the .gitattributes files in that directory, and save it
# as GIT_DIR/info/attributes, and then get `git check-attr'
# to read it for us.
git.show('%s:%s' % (commit_rev, gitattributes_file),
_outfile=BARE_REPO_ATTRIBUTES_FILE)
attr_value = get_attribute(rel_file_path, attr_name)
# If this .gitattribute file provided us with an attribute
# value, then we're done.
if attr_value != UNSPECIFIED_ATTR:
break
if not path_to_rel or path_to_rel == '.':
# No more parent directories. We're done.
break
(path_to_rel, parent_dir) = os.path.split(path_to_rel)
rel_file_path = os.path.join(parent_dir, rel_file_path)
# If none of the .gitattributes files in the project provided
# an attribute value, then check the `info/default_attributes'
# file.
if attr_value == UNSPECIFIED_ATTR and isfile(DEFAULT_ATTRIBUTES_FILE):
copy(DEFAULT_ATTRIBUTES_FILE, BARE_REPO_ATTRIBUTES_FILE)
attr_value = get_attribute(filename, attr_name)
return attr_value
def get_standard_commit_email(self, commit):
"""Return an Email object for the given commit.
Here, "standard" means that the Email returned corresponds
to the Email the git-hooks sends by default, before any
possible project-specific customization is applied.
Before sending this email, users of this method are expected
to apply those customizations as needed.
PARAMETERS
commit: A CommitInfo object.
"""
subject_prefix = commit_email_subject_prefix(
self.email_info.project_name,
self.ref_name,
)
subject = f"{subject_prefix} {commit.subject[:SUBJECT_MAX_SUBJECT_CHARS]}"
# Generate the body of the email in two pieces:
# 1. The commit description without the patch;
# 2. The diff stat and patch.
# This allows us to insert our little "Diff:" marker that
# bugtool detects when parsing the email for filing (this
# part is now performed by the Email class). The purpose
# is to prevent bugtool from searching for TNs in the patch
# itself.
body = git.log(commit.rev, max_count="1", _decode=True) + "\n"
if git_config("hooks.commit-url") is not None:
url_info = {"rev": commit.rev, "ref_name": self.ref_name}
body = git_config("hooks.commit-url") % url_info + "\n\n" + body
if git_config("hooks.disable-email-diff"):
diff = None
else:
diff = git.show(commit.rev, p=True, M=True, stat=True, pretty="format:")
# Decode the diff line-by-line:
#
# It seems conceivable that the diff may cover multiple files,
# and that the files may have different encodings, so we cannot
# assume that the entire output follows the same encoding.
diff = safe_decode_by_line(diff)
# Add a small "---" separator line at the beginning of
# the diff section we just computed. This mimicks what
# "git show" would do if we hadn't provided an empty
# "format:" string to the "--pretty" command-line option.
diff = "---\n" + diff
filer_cmd = git_config("hooks.file-commit-cmd")
if filer_cmd is not None:
filer_cmd = shlex.split(filer_cmd)
email_bcc = git_config("hooks.filer-email")
return Email(
self.email_info,
commit.email_to(self.ref_name),
email_bcc,
subject,
body,
commit.full_author_email,
self.ref_name,
commit.base_rev_for_display(),
commit.rev,
diff,
filer_cmd=filer_cmd,
)
