def get_repository_info(self):
if not check_install('cm version'):
return None
# Get the workspace directory, so we can strip it from the diff output
self.workspacedir = execute(["cm", "gwp", ".", "--format={1}"],
split_lines=False,
ignore_errors=True).strip()
logging.debug("Workspace is %s" % self.workspacedir)
# Get the repository that the current directory is from
split = execute(["cm", "ls", self.workspacedir, "--format={8}"], split_lines=True,
ignore_errors=True)
# remove blank lines
split = filter(None, split)
m = re.search(r'^rep:(.+)$', split[0], re.M)
if not m:
return None
path = m.group(1)
return RepositoryInfo(path,
supports_changesets=True,
supports_parent_diffs=False)
def check_gnu_patch():
"""Checks if GNU patch is installed, and informs the user if it's not."""
has_gnu_patch = False
try:
result = execute(['patch', '--version'], ignore_errors=True)
has_gnu_patch = 'Free Software Foundation' in result
except OSError:
pass
try:
# SunOS has gpatch
result = execute(['gpatch', '--version'], ignore_errors=True)
has_gnu_patch = 'Free Software Foundation' in result
except OSError:
pass
if not has_gnu_patch:
sys.stderr.write('\n')
sys.stderr.write('GNU patch is required to post this review.'
'Make sure it is installed and in the path.\n')
sys.stderr.write('\n')
if os.name == 'nt':
sys.stderr.write('On Windows, you can install this from:\n')
sys.stderr.write(GNU_PATCH_WIN32_URL)
sys.stderr.write('\n')
die()
def make_diff(self, ancestor, commit=""):
"""
Performs a diff on a particular branch range.
"""
# Use `git diff ancestor..commit` by default, except from when there
# is no `commit` given - then use `git show ancestor`.
if commit:
diff_command = 'diff'
rev_range = "%s..%s" % (ancestor, commit)
else:
diff_command = 'show'
rev_range = ancestor
if self.type == "svn":
diff_lines = execute([self.git, diff_command, "--no-color",
"--no-prefix", "--no-ext-diff", "-r", "-u",
rev_range],
split_lines=True)
return self.make_svn_diff(ancestor, diff_lines)
elif self.type == "git":
cmdline = [self.git, diff_command, "--no-color", "--full-index",
"--no-ext-diff", "--ignore-submodules", "--no-renames",
rev_range]
if (self.capabilities is not None and
self.capabilities.has_capability('diffs', 'moved_files')):
cmdline.append('-M')
return execute(cmdline)
return None
def make_diff(self, ancestor, commit=""):
"""
Performs a diff on a particular branch range.
"""
if commit:
rev_range = "%s..%s" % (ancestor, commit)
else:
rev_range = ancestor
if self.type == "svn":
diff_lines = execute([self.git, "diff", "--no-color",
"--no-prefix", "--no-ext-diff", "-r", "-u",
rev_range],
split_lines=True)
return self.make_svn_diff(ancestor, diff_lines)
elif self.type == "git":
cmdline = [self.git, "diff", "--no-color", "--full-index",
"--no-ext-diff", "--ignore-submodules", "--no-renames",
rev_range]
if (self.capabilities is not None and
self.capabilities.has_capability('diffs', 'moved_files')):
cmdline.append('-M')
return execute(cmdline)
return None
def diff(self, args):
"""
Performs a diff across all modified files in the branch, taking into
account a parent branch.
"""
parent_branch = self._options.parent_branch
self.merge_base = execute([self.git, "merge-base",
self.upstream_branch,
self.head_ref]).strip()
if parent_branch:
diff_lines = self.make_diff(parent_branch)
parent_diff_lines = self.make_diff(self.merge_base, parent_branch)
else:
diff_lines = self.make_diff(self.merge_base, self.head_ref)
parent_diff_lines = None
if self._options.guess_summary and not self._options.summary:
s = execute([self.git, "log", "--pretty=format:%s", "HEAD^.."],
ignore_errors=True)
self._options.summary = s.replace('\n', ' ').strip()
if self._options.guess_description and not self._options.description:
self._options.description = execute(
[self.git, "log", "--pretty=format:%s%n%n%b",
(parent_branch or self.merge_base) + ".."],
ignore_errors=True).strip()
return (diff_lines, parent_diff_lines)
def diff_files(self, old_file, new_file):
"""Return unified diff for file.
Most effective and reliable way is use gnu diff.
"""
diff_cmd = ["diff", "-uN", old_file, new_file]
dl = execute(diff_cmd, extra_ignore_errors=(1,2),
translate_newlines=False)
# If the input file has ^M characters at end of line, lets ignore them.
dl = dl.replace('\r\r\n', '\r\n')
dl = dl.splitlines(True)
# Special handling for the output of the diff tool on binary files:
# diff outputs "Files a and b differ"
# and the code below expects the output to start with
# "Binary files "
if (len(dl) == 1 and
dl[0].startswith('Files %s and %s differ' % (old_file, new_file))):
dl = ['Binary files %s and %s differ\n' % (old_file, new_file)]
# We need oids of files to translate them to paths on reviewboard repository
old_oid = execute(["cleartool", "describe", "-fmt", "%On", old_file])
new_oid = execute(["cleartool", "describe", "-fmt", "%On", new_file])
if dl == [] or dl[0].startswith("Binary files "):
if dl == []:
dl = ["File %s in your changeset is unmodified\n" % new_file]
dl.insert(0, "==== %s %s ====\n" % (old_oid, new_oid))
dl.append('\n')
else:
dl.insert(2, "==== %s %s ====\n" % (old_oid, new_oid))
return dl
def do_diff(self, changeset):
"""Generates a unified diff for all files in the changeset."""
diff = []
for old_file, new_file, xpatches in changeset:
# We need oids of files to translate them to paths on
# reviewboard repository
old_oid = execute(["cleartool", "describe", "-fmt", "%On",
old_file])
new_oid = execute(["cleartool", "describe", "-fmt", "%On",
new_file])
dl = self._diff(old_file, new_file, xpatches=xpatches)
oid_line = "==== %s %s ====" % (old_oid, new_oid)
if dl is None:
dl = [oid_line,
'Binary files %s and %s differ' % (old_file, new_file),
'']
elif not dl:
dl = [oid_line,
'File %s in your changeset is unmodified' % new_file,
'']
else:
dl.insert(2, oid_line)
diff.append(os.linesep.join(dl))
return (''.join(diff), None)
def test_scanning_nested_repos_2(self):
"""Testing scan_for_usable_client with nested repositories (svn inside
git)
"""
git_dir = os.path.join(self.testdata_dir, 'git-repo')
svn_dir = os.path.join(self.testdata_dir, 'svn-repo')
# Check out git first
clone_dir = self.chdir_tmp()
git_clone_dir = os.path.join(clone_dir, 'git-repo')
os.mkdir(git_clone_dir)
execute(['git', 'clone', git_dir, git_clone_dir],
env=None, ignore_errors=False, extra_ignore_errors=())
# Now check out svn.
svn_clone_dir = os.path.join(git_clone_dir, 'svn-repo')
os.chdir(git_clone_dir)
execute(['svn', 'co', 'file://%s' % svn_dir, 'svn-repo'],
env=None, ignore_errors=False, extra_ignore_errors=())
os.chdir(svn_clone_dir)
repository_info, tool = scan_usable_client({}, self.options)
self.assertEqual(repository_info.local_path,
os.path.realpath(svn_clone_dir))
self.assertEqual(type(tool), SVNClient)
def get_repository_info(self):
if not check_install('cm version'):
return None
# Get the repository that the current directory is from. If there
# is more than one repository mounted in the current directory,
# bail out for now (in future, should probably enter a review
# request per each repository.)
split = execute(["cm", "ls", "--format={8}"], split_lines=True,
ignore_errors=True)
m = re.search(r'^rep:(.+)$', split[0], re.M)
if not m:
return None
# Make sure the repository list contains only one unique entry
if len(split) != split.count(split[0]):
# Not unique!
die('Directory contains more than one mounted repository')
path = m.group(1)
# Get the workspace directory, so we can strip it from the diff output
self.workspacedir = execute(["cm", "gwp", ".", "--format={1}"],
split_lines=False,
ignore_errors=True).strip()
logging.debug("Workspace is %s" % self.workspacedir)
return RepositoryInfo(path,
supports_changesets=True,
supports_parent_diffs=False)
def diff_directories(self, old_dir, new_dir):
"""Return uniffied diff between two directories content.
Function save two version's content of directory to temp
files and treate them as casual diff between two files.
"""
old_content = self._directory_content(old_dir)
new_content = self._directory_content(new_dir)
old_tmp = make_tempfile(content=old_content)
new_tmp = make_tempfile(content=new_content)
diff_cmd = ["diff", "-uN", old_tmp, new_tmp]
dl = execute(diff_cmd,
extra_ignore_errors=(1, 2),
translate_newlines=False,
split_lines=True)
# Replacing temporary filenames to
# real directory names and add ids
if dl:
dl[0] = dl[0].replace(old_tmp, old_dir)
dl[1] = dl[1].replace(new_tmp, new_dir)
old_oid = execute(["cleartool", "describe", "-fmt", "%On",
old_dir])
new_oid = execute(["cleartool", "describe", "-fmt", "%On",
new_dir])
dl.insert(2, "==== %s %s ====\n" % (old_oid, new_oid))
return dl
def diff_between_revisions(self, revision_range, args, repository_info):
"""
Performs a diff between 2 revisions of a Mercurial repository.
"""
if self._type != 'hg':
raise NotImplementedError
if ':' in revision_range:
r1, r2 = revision_range.split(':')
else:
# If only 1 revision is given, we find the first parent and use
# that as the second revision.
#
# We could also use "hg diff -c r1", but then we couldn't reuse the
# code for extracting descriptions.
r2 = revision_range
r1 = execute(["hg", "parents", "-r", r2,
"--template", "{rev}\n"]).split()[0]
if self._options.guess_summary and not self._options.summary:
self._options.summary = self.extract_summary(r2)
if self._options.guess_description and not self._options.description:
self._options.description = self.extract_description(r1, r2)
return (execute(["hg", "diff", "-r", r1, "-r", r2],
env=self._hg_env), None)
def _apply_patch_for_empty_files(self, patch, p_num):
"""Returns True if any empty files in the patch are applied.
If there are no empty files in the patch or if an error occurs while
applying the patch, we return False.
"""
patched_empty_files = False
added_files = re.findall(r"^Index:\s+(\S+)\t\(added\)$", patch, re.M)
deleted_files = re.findall(r"^Index:\s+(\S+)\t\(deleted\)$", patch, re.M)
if added_files:
added_files = self._strip_p_num_slashes(added_files, int(p_num))
make_empty_files(added_files)
result = execute(["svn", "add"] + added_files, ignore_errors=True, none_on_ignored_error=True)
if result is None:
logging.error('Unable to execute "svn add" on: %s', ", ".join(added_files))
else:
patched_empty_files = True
if deleted_files:
deleted_files = self._strip_p_num_slashes(deleted_files, int(p_num))
result = execute(["svn", "delete"] + deleted_files, ignore_errors=True, none_on_ignored_error=True)
if result is None:
logging.error('Unable to execute "svn delete" on: %s', ", ".join(deleted_files))
else:
patched_empty_files = True
return patched_empty_files
def merge(self, target, destination, message, author, squash=False,
run_editor=False):
"""Merges the target branch with destination branch."""
rc, output = execute(
['git', 'checkout', destination],
ignore_errors=True,
return_error_code=True)
if rc:
raise MergeError("Could not checkout to branch '%s'.\n\n%s" %
(destination, output))
if squash:
method = '--squash'
else:
method = '--no-ff'
rc, output = execute(
['git', 'merge', target, method, '--no-commit'],
ignore_errors=True,
return_error_code=True)
if rc:
raise MergeError("Could not merge branch '%s' into '%s'.\n\n%s" %
(target, destination, output))
self.create_commit(message, author, run_editor)
def _set_label(self, label, path):
"""Set a ClearCase label on elements seen under path.
Args:
label (unicode):
The label to set.
path (unicode):
The filesystem path to set the label on.
"""
checkedout_elements = self._list_checkedout(path)
if checkedout_elements:
raise Exception(
'ClearCase backend cannot set label when some elements are '
'checked out:\n%s' % ''.join(checkedout_elements))
# First create label in vob database.
execute(['cleartool', 'mklbtype', '-c', 'label created for rbtools',
label],
with_errors=True)
# We ignore return code 1 in order to omit files that ClearCase cannot
# read.
recursive_option = ''
if cpath.isdir(path):
recursive_option = '-recurse'
# Apply label to path.
execute(['cleartool', 'mklabel', '-nc', recursive_option, label, path],
extra_ignore_errors=(1,),
with_errors=False)
def make_svn_diff(self, parent_branch, diff_lines):
"""
Formats the output of git diff such that it's in a form that
svn diff would generate. This is needed so the SVNTool in Review
Board can properly parse this diff.
"""
rev = execute([self.git, "svn", "find-rev", parent_branch]).strip()
if not rev and self.merge_base:
rev = execute([self.git, "svn", "find-rev",
self.merge_base]).strip()
if not rev:
return None
diff_data = ""
filename = ""
newfile = False
for line in diff_lines:
if line.startswith("diff "):
# Grab the filename and then filter this out.
# This will be in the format of:
#
# diff --git a/path/to/file b/path/to/file
info = line.split(" ")
diff_data += "Index: %s\n" % info[2]
diff_data += "=" * 67
diff_data += "\n"
elif line.startswith("index "):
# Filter this out.
pass
elif line.strip() == "--- /dev/null":
# New file
newfile = True
elif line.startswith("--- "):
newfile = False
diff_data += "--- %s\t(revision %s)\n" % \
(line[4:].strip(), rev)
elif line.startswith("+++ "):
filename = line[4:].strip()
if newfile:
diff_data += "--- %s\t(revision 0)\n" % filename
diff_data += "+++ %s\t(revision 0)\n" % filename
else:
# We already printed the "--- " line.
diff_data += "+++ %s\t(working copy)\n" % filename
elif line.startswith("new file mode"):
# Filter this out.
pass
elif line.startswith("Binary files "):
# Add the following so that we know binary files were
# added/changed.
diff_data += "Cannot display: file marked as a binary type.\n"
diff_data += "svn:mime-type = application/octet-stream\n"
else:
diff_data += line
return diff_data
def update_commits_with_reviewer_info(self, options, review_url):
""" Use git commit --amend to add Reviewed-by: to each commit """
num_successful_updates = 0
if not self.rev_range_for_diff:
return 0 # don't know what commits to update, so bail.
# Get the list of commits we want to update.
commits = execute([self.git, "rev-list", "--reverse",
"..".join(self.rev_range_for_diff)],
ignore_errors=True, none_on_ignored_error=True,
split_lines=True)
if not commits:
return 0 # illegal commit-range
commits = [c.strip() for c in commits]
reviewed_by = "Reviewed-By:"
if options.target_people:
reviewed_by += " " + options.target_people
if options.target_people and options.target_groups:
reviewed_by += " and"
if options.target_groups:
reviewed_by += " groups:" + options.target_groups
reviewed_by += " <%s>" % review_url
# When post-review is run with -r, the first commits in our
# revrange will have already been updated with reviewer info
# (from previous calls to post-review for this review). In
# fact, the early commits may even have better reviewed-by
# information than we do, since they are more likely to have
# had the reviewers specified on the commandline when
# post-review was run. So if we see an existing commit with
# reviewed-by info, prefer that to our own reviewed-by text.
output = execute([self.git, "notes", "show", commits[0]],
with_errors=True, ignore_errors=True,
none_on_ignored_error=True)
if output:
for line in output.splitlines():
if line.startswith('Reviewed-By: ') and review_url in line:
reviewed_by = line
# We don't need to update this commit because we
# know it's reviewed-by text is already "right".
commits = commits[1:] # small optimization
num_successful_updates += 1 # count as a "null update"
break
# TODO(csilvers): shell-escape any nasty characters.
# Use perl to delete any old Reviewed-By messages and insert a new one
perlcmd = ("print unless /Reviewed-By: /i; "
"if (eof) { print; print q{%s} }" % reviewed_by)
git_editor_cmd = r'sh -c "perl -nli -e \"%s\" \"$1\""' % perlcmd
for commit in commits:
output = execute([self.git, "notes", "edit", commit],
env={"GIT_EDITOR": git_editor_cmd},
with_errors=True, ignore_errors=True,
none_on_ignored_error=True)
if output is not None:
num_successful_updates += 1
return num_successful_updates
def _get_outgoing_diff(self, files):
"""
When working with a clone of a Mercurial remote, we need to find
out what the outgoing revisions are for a given branch. It would
be nice if we could just do `hg outgoing --patch <remote>`, but
there are a couple of problems with this.
For one, the server-side diff parser isn't yet equipped to filter out
diff headers such as "comparing with..." and "changeset: <rev>:<hash>".
Another problem is that the output of `outgoing` potentially includes
changesets across multiple branches.
In order to provide the most accurate comparison between one's local
clone and a given remote -- something akin to git's diff command syntax
`git diff <treeish>..<treeish>` -- we have to do the following:
- get the name of the current branch
- get a list of outgoing changesets, specifying a custom format
- filter outgoing changesets by the current branch name
- get the "top" and "bottom" outgoing changesets
- use these changesets as arguments to `hg diff -r <rev> -r <rev>`
Future modifications may need to be made to account for odd cases like
having multiple diverged branches which share partial history -- or we
can just punish developers for doing such nonsense :)
"""
files = files or []
remote = self._remote_path[0]
if not remote and self._options.parent_branch:
remote = self._options.parent_branch
current_branch = execute(['hg', 'branch'], env=self._hg_env).strip()
outgoing_changesets = \
self._get_outgoing_changesets(current_branch, remote)
top_rev, bottom_rev = \
self._get_top_and_bottom_outgoing_revs(outgoing_changesets) \
if len(outgoing_changesets) > 0 else (None, None)
if self._options.guess_summary and not self._options.summary:
self._options.summary = self.extract_summary(top_rev).rstrip("\n")
if self._options.guess_description and not self._options.description:
self._options.description = self.extract_description(bottom_rev,
top_rev)
if bottom_rev is not None and top_rev is not None:
full_command = ['hg', 'diff', '-r', str(bottom_rev), '-r',
str(top_rev)] + files
return (execute(full_command, env=self._hg_env), None)
else:
return ("", None)
def _write_file(self, depot_path, tmpfile):
"""
Grabs a file from Perforce and writes it to a temp file. p4 print sets
the file readonly and that causes a later call to unlink fail. So we
make the file read/write.
"""
logging.debug('Writing "%s" to "%s"' % (depot_path, tmpfile))
execute(["p4", "print", "-o", tmpfile, "-q", depot_path])
os.chmod(tmpfile, stat.S_IREAD | stat.S_IWRITE)
def _svn_add_file_commit(self, filename, data, msg, add_file=True):
outfile = open(filename, 'w')
outfile.write(data)
outfile.close()
if add_file:
execute(['svn', 'add', filename], ignore_errors=True)
execute(['svn', 'commit', '-m', msg])
def apply_patch_for_empty_files(self, patch, p_num, revert=False):
"""Return whether any empty files in the patch are applied.
Args:
patch (bytes):
The contents of the patch.
p_num (unicode):
The prefix level of the diff.
revert (bool, optional):
Whether the patch should be reverted rather than applied.
Returns:
``True`` if there are empty files in the patch. ``False`` if there
were no empty files, or if an error occurred while applying the
patch.
"""
patched_empty_files = False
added_files = re.findall(r'--- %s\t%s\n'
r'\+\+\+ b/(\S+)\t[^\r\n\t\f]+\n'
r'(?:[^@]|$)'
% (self.PRE_CREATION,
re.escape(self.PRE_CREATION_DATE)), patch)
deleted_files = re.findall(r'--- a/(\S+)\t[^\r\n\t\f]+\n'
r'\+\+\+ %s\t%s\n'
r'(?:[^@]|$)'
% (self.PRE_CREATION,
re.escape(self.PRE_CREATION_DATE)),
patch)
if added_files:
added_files = self._strip_p_num_slashes(added_files, int(p_num))
make_empty_files(added_files)
result = execute(['hg', 'add'] + added_files, ignore_errors=True,
none_on_ignored_error=True)
if result is None:
logging.error('Unable to execute "hg add" on: %s',
', '.join(added_files))
else:
patched_empty_files = True
if deleted_files:
deleted_files = self._strip_p_num_slashes(deleted_files,
int(p_num))
result = execute(['hg', 'remove'] + deleted_files,
ignore_errors=True, none_on_ignored_error=True)
if result is None:
logging.error('Unable to execute "hg remove" on: %s',
', '.join(deleted_files))
else:
patched_empty_files = True
return patched_empty_files
def get_repository_info(self):
"""Returns information on the Clear Case repository.
This will first check if the cleartool command is
installed and in the path, and post-review was run
from inside of the view.
"""
if not check_install('cleartool help'):
return None
viewname = execute(["cleartool", "pwv", "-short"]).strip()
if viewname.startswith('** NONE'):
return None
# Now that we know it's ClearCase, make sure we have GNU diff installed,
# and error out if we don't.
check_gnu_diff()
property_lines = execute(["cleartool", "lsview", "-full", "-properties",
"-cview"], split_lines=True)
for line in property_lines:
properties = line.split(' ')
if properties[0] == 'Properties:':
# Determine the view type and check if it's supported.
#
# Specifically check if webview was listed in properties
# because webview types also list the 'snapshot'
# entry in properties.
if 'webview' in properties:
die("Webviews are not supported. You can use post-review"
" only in dynamic or snapshot view.")
if 'dynamic' in properties:
self.viewtype = 'dynamic'
else:
self.viewtype = 'snapshot'
break
# Find current VOB's tag
vobstag = execute(["cleartool", "describe", "-short", "vob:."],
ignore_errors=True).strip()
if "Error: " in vobstag:
die("To generate diff run post-review inside vob.")
# From current working directory cut path to VOB.
# VOB's tag contain backslash character before VOB's name.
# I hope that first character of VOB's tag like '\new_proj'
# won't be treat as new line character but two separate:
# backslash and letter 'n'
cwd = os.getcwd()
base_path = cwd[:cwd.find(vobstag) + len(vobstag)]
return ClearCaseRepositoryInfo(path=base_path,
base_path=base_path,
vobstag=vobstag,
supports_parent_diffs=False)
def make_perforce_diff(self, merge_base, diff_lines):
"""Format the output of git diff to look more like perforce's."""
diff_data = ''
filename = ''
p4rev = ''
# Find which depot changelist we're based on
log = execute([self.git, 'log', merge_base], ignore_errors=True)
for line in log:
m = re.search(r'[rd]epo.-paths = "(.+)": change = (\d+).*\]',
log, re.M)
if m:
base_path = m.group(1).strip()
p4rev = m.group(2).strip()
break
else:
# We should really raise an error here, base_path is required
pass
for i, line in enumerate(diff_lines):
if line.startswith('diff '):
# Grab the filename and then filter this out.
# This will be in the format of:
# diff --git a/path/to/file b/path/to/file
filename = line.split(' ')[2].strip()
elif (line.startswith('index ') or
line.startswith('new file mode ')):
# Filter this out
pass
elif (line.startswith('--- ') and i + 1 < len(diff_lines) and
diff_lines[i + 1].startswith('+++ ')):
data = execute(
['p4', 'files', base_path + filename + '@' + p4rev],
ignore_errors=True)
m = re.search(r'^%s%s#(\d+).*$' % (re.escape(base_path),
re.escape(filename)),
data, re.M)
if m:
fileVersion = m.group(1).strip()
else:
fileVersion = 1
diff_data += '--- %s%s\t%s%s#%s\n' % (base_path, filename,
base_path, filename,
fileVersion)
elif line.startswith('+++ '):
# TODO: add a real timestamp
diff_data += '+++ %s%s\t%s\n' % (base_path, filename,
'TIMESTAMP')
else:
diff_data += line
return diff_data
def delete_branch(self, branch_name, merged_only=True):
"""Deletes the specified branch.
If merged_only is False, then the branch will be deleted even if not
yet merged into an upstream branch.
"""
if merged_only:
delete_flag = '-d'
else:
delete_flag = '-D'
execute(['git', 'branch', delete_flag, branch_name])
def create_commit(self, message, author, files=[], all_files=False):
"""Commits the given modified files.
This is expected to be called after applying a patch. This commits the
patch using information from the review request, opening the commit
message in $EDITOR to allow the user to update it.
"""
modified_message = edit_text(message)
hg_command = ['hg', 'commit', '-m', modified_message,
'-u %s <%s>' % (author.fullname, author.email)]
execute(hg_command + files)
def _remove_label(self, label):
"""Remove a ClearCase label from vob database.
It will remove all references of this label on elements.
"""
# Be sure label is prefix by lbtype.
if not label.startswith(self.REVISION_LABEL_PREFIX):
label = '%s%s' % (self.REVISION_LABEL_PREFIX, label)
# Label exists so remove it.
execute(['cleartool', 'rmtype', '-rmall', '-force', label],
with_errors=True)
def _get_hgsubversion_diff(self, files):
parent = execute(['hg', 'parent', '--svn', '--template',
'{node}\n']).strip()
if self._options.parent_branch:
parent = self._options.parent_branch
if self._options.guess_summary and not self._options.summary:
self._options.summary = self.extract_summary(".")
if self._options.guess_description and not self._options.description:
self._options.description = self.extract_description(parent, ".")
return (execute(["hg", "diff", "--svn", '-r%s:.' % parent]), None)
def extract_description(self, rev1, rev2):
"""
Extracts all descriptions in the given revision range and concatenates
them, most recent ones going first.
"""
numrevs = len(execute([
'hg', 'log', '-r%s:%s' % (rev2, rev1),
'--follow', '--template', r'{rev}\n'], env=self._hg_env
).strip().split('\n'))
return execute(['hg', 'log', '-r%s:%s' % (rev2, rev1),
'--follow', '--template',
r'{desc}\n\n', '--limit',
str(numrevs - 1)], env=self._hg_env).strip()
def diff_between_revisions(self, revision_range, args, repository_info):
"""Perform a diff between two arbitrary revisions"""
head_ref = "HEAD"
if self.head_ref:
head_ref = self.head_ref
# Make a parent diff to the first of the revisions so that we
# never end up with broken patches:
self.merge_base = execute([self.git, "merge-base",
self.upstream_branch,
head_ref]).strip()
if ":" not in revision_range:
# only one revision is specified, diff against its
# immediate parent
revision_range = revision_range + "^:" + revision_range
# Get revision 1 and 2 from the range.
r1, r2 = revision_range.split(":")
# Check if parent contains the first revision and make a
# parent diff if not:
pdiff_required = execute([self.git, "branch", "-r",
"--contains", r1])
parent_diff_lines = None
if not pdiff_required:
parent_diff_lines = self.make_diff(self.merge_base, r1)
if self.options.guess_summary and not self.options.summary:
self.options.summary = execute(
[self.git, "log", "--pretty=format:%s", "%s^!" % r2],
ignore_errors=True).strip()
if (self.options.guess_description and
not self.options.description):
self.options.description = execute(
[self.git, "log", "--pretty=format:%s%n%n%b",
"%s..%s" % (r1, r2)],
ignore_errors=True).strip()
diff_lines = self.make_diff(r1, r2)
return {
'diff': diff_lines,
'parent_diff_lines': parent_diff_lines,
'base_commit_id': self.merge_base,
}
def _get_hgsubversion_diff(self, files):
parent = execute(["hg", "parent", "--svn", "--template", "{node}\n"]).strip()
if self.options.parent_branch:
parent = self.options.parent_branch
if self.options.guess_summary and not self.options.summary:
self.options.summary = self.extract_summary(".")
if self.options.guess_description and not self.options.description:
self.options.description = self.extract_description(parent, ".")
rs = "-r{0}:{1}".format(parent, files[0] if len(files) == 1 else ".")
return (execute(["hg", "diff", "--svn", rs]), None)
def scan_for_server(self, repository_info):
# Scan first for dot files, since it's faster and will cover the
# user's $HOME/.reviewboardrc
server_url = super(GitClient, self).scan_for_server(repository_info)
if server_url:
return server_url
# TODO: Maybe support a server per remote later? Is that useful?
url = execute([self.git, "config", "--get", "reviewboard.url"],
ignore_errors=True).strip()
if url:
return url
if self.type == "svn":
# Try using the reviewboard:url property on the SVN repo, if it
# exists.
prop = SVNClient().scan_for_server_property(repository_info)
if prop:
return prop
elif self.type == 'perforce':
prop = PerforceClient().scan_for_server(repository_info)
if prop:
return prop
return None
def get_repository_info(self):
"""Get repository information for the current Git working tree.
This function changes the directory to the top level directory of the
current working tree.
"""
if not check_install(['git', '--help']):
# CreateProcess (launched via subprocess, used by check_install)
# does not automatically append .cmd for things it finds in PATH.
# If we're on Windows, and this works, save it for further use.
if (sys.platform.startswith('win')
and check_install(['git.cmd', '--help'])):
self.git = 'git.cmd'
else:
logging.debug('Unable to execute "git --help" or "git.cmd '
'--help": skipping Git')
return None
git_dir = execute([self.git, "rev-parse", "--git-dir"],
ignore_errors=True).rstrip("\n")
if git_dir.startswith("fatal:") or not os.path.isdir(git_dir):
return None
# Sometimes core.bare is not set, and generates an error, so ignore
# errors. Valid values are 'true' or '1'.
bare = execute([self.git, 'config', 'core.bare'],
ignore_errors=True).strip()
self.bare = bare in ('true', '1')
# If we are not working in a bare repository, then we will change
# directory to the top level working tree lose our original position.
# However, we need the original working directory for file exclusion
# patterns, so we save it here.
if self._original_cwd is None:
self._original_cwd = os.getcwd()
# Running in directories other than the top level of
# of a work-tree would result in broken diffs on the server
if not self.bare:
git_top = execute([self.git, "rev-parse", "--show-toplevel"],
ignore_errors=True).rstrip("\n")
# Top level might not work on old git version se we use git dir
# to find it.
if (git_top.startswith('fatal:') or not os.path.isdir(git_dir)
or git_top.startswith('cygdrive')):
git_top = git_dir
os.chdir(os.path.abspath(git_top))
self.head_ref = execute([self.git, 'symbolic-ref', '-q', 'HEAD'],
ignore_errors=True).strip()
# We know we have something we can work with. Let's find out
# what it is. We'll try SVN first, but only if there's a .git/svn
# directory. Otherwise, it may attempt to create one and scan
# revisions, which can be slow. Also skip SVN detection if the git
# repository was specified on command line.
git_svn_dir = os.path.join(git_dir, 'svn')
if (not getattr(self.options, 'repository_url', None)
and os.path.isdir(git_svn_dir)
and len(os.listdir(git_svn_dir)) > 0):
data = execute([self.git, "svn", "info"], ignore_errors=True)
m = re.search(r'^Repository Root: (.+)$', data, re.M)
if m:
path = m.group(1)
m = re.search(r'^URL: (.+)$', data, re.M)
if m:
base_path = m.group(1)[len(path):] or "/"
m = re.search(r'^Repository UUID: (.+)$', data, re.M)
if m:
uuid = m.group(1)
self.type = "svn"
# Get SVN tracking branch
if getattr(self.options, 'tracking', None):
self.upstream_branch = self.options.tracking
else:
data = execute([self.git, "svn", "rebase", "-n"],
ignore_errors=True)
m = re.search(r'^Remote Branch:\s*(.+)$', data,
re.M)
if m:
self.upstream_branch = m.group(1)
else:
sys.stderr.write('Failed to determine SVN '
'tracking branch. Defaulting'
'to "master"\n')
self.upstream_branch = 'master'
return SVNRepositoryInfo(path=path,
base_path=base_path,
uuid=uuid,
supports_parent_diffs=True)
else:
# Versions of git-svn before 1.5.4 don't (appear to) support
# 'git svn info'. If we fail because of an older git install,
# here, figure out what version of git is installed and give
# the user a hint about what to do next.
version = execute([self.git, "svn", "--version"],
ignore_errors=True)
version_parts = re.search('version (\d+)\.(\d+)\.(\d+)',
version)
svn_remote = execute(
[self.git, "config", "--get", "svn-remote.svn.url"],
ignore_errors=True)
if (version_parts and svn_remote and not is_valid_version(
(int(version_parts.group(1)), int(version_parts.group(2)),
int(version_parts.group(3))), (1, 5, 4))):
raise SCMError('Your installation of git-svn must be '
'upgraded to version 1.5.4 or later.')
# Okay, maybe Perforce (git-p4).
git_p4_ref = os.path.join(git_dir, 'refs', 'remotes', 'p4', 'master')
if os.path.exists(git_p4_ref):
data = execute([self.git, 'config', '--get', 'git-p4.port'],
ignore_errors=True)
m = re.search(r'(.+)', data)
if m:
port = m.group(1)
else:
port = os.getenv('P4PORT')
if port:
self.type = 'perforce'
self.upstream_branch = 'remotes/p4/master'
return RepositoryInfo(path=port,
base_path='',
supports_parent_diffs=True)
# Nope, it's git then.
# Check for a tracking branch and determine merge-base
self.upstream_branch = ''
if self.head_ref:
short_head = self._strip_heads_prefix(self.head_ref)
merge = execute(
[self.git, 'config', '--get',
'branch.%s.merge' % short_head],
ignore_errors=True).strip()
remote = execute(
[self.git, 'config', '--get',
'branch.%s.remote' % short_head],
ignore_errors=True).strip()
merge = self._strip_heads_prefix(merge)
if remote and remote != '.' and merge:
self.upstream_branch = '%s/%s' % (remote, merge)
url = None
if getattr(self.options, 'repository_url', None):
url = self.options.repository_url
self.upstream_branch = self.get_origin(self.upstream_branch,
True)[0]
else:
self.upstream_branch, origin_url = \
self.get_origin(self.upstream_branch, True)
if not origin_url or origin_url.startswith("fatal:"):
self.upstream_branch, origin_url = self.get_origin()
url = origin_url.rstrip('/')
# Central bare repositories don't have origin URLs.
# We return git_dir instead and hope for the best.
if not url:
url = os.path.abspath(git_dir)
# There is no remote, so skip this part of upstream_branch.
self.upstream_branch = self.upstream_branch.split('/')[-1]
if url:
self.type = "git"
return RepositoryInfo(path=url,
base_path='',
supports_parent_diffs=True)
return None
def diff_between_revisions(self, revision_range, args, repository_info):
"""Perform a diff between two arbitrary revisions"""
head_ref = "HEAD"
if self.head_ref:
head_ref = self.head_ref
# Make a parent diff to the first of the revisions so that we
# never end up with broken patches:
self.merge_base = execute(
[self.git, "merge-base", self.upstream_branch, head_ref]).strip()
if ":" not in revision_range:
# only one revision is specified
# Check if parent contains the first revision and make a
# parent diff if not:
pdiff_required = execute(
[self.git, "branch", "-r", "--contains", revision_range])
parent_diff_lines = None
if not pdiff_required:
parent_diff_lines = self.make_diff(self.merge_base,
revision_range)
if self.options.guess_summary and not self.options.summary:
s = execute([
self.git, "log", "--pretty=format:%s",
revision_range + ".."
],
ignore_errors=True)
self.options.summary = s.replace('\n', ' ').strip()
if (self.options.guess_description
and not self.options.description):
self.options.description = execute([
self.git, "log", "--pretty=format:%s%n%n%b",
revision_range + ".."
],
ignore_errors=True).strip()
return (self.make_diff(revision_range), parent_diff_lines)
else:
r1, r2 = revision_range.split(":")
# Check if parent contains the first revision and make a
# parent diff if not:
pdiff_required = execute(
[self.git, "branch", "-r", "--contains", r1])
parent_diff_lines = None
if not pdiff_required:
parent_diff_lines = self.make_diff(self.merge_base, r1)
if self.options.guess_summary and not self.options.summary:
s = execute([
self.git, "log", "--pretty=format:%s",
"%s..%s" % (r1, r2)
],
ignore_errors=True)
self.options.summary = s.replace('\n', ' ').strip()
if (self.options.guess_description
and not self.options.description):
self.options.description = execute([
self.git, "log", "--pretty=format:%s%n%n%b",
"%s..%s" % (r1, r2)
],
ignore_errors=True).strip()
return (self.make_diff(r1, r2), parent_diff_lines)
def _svn_add_file_commit(self, filename, data, msg):
outfile = open(filename, 'w')
outfile.write(data)
outfile.close()
execute(['svn', 'add', filename])
execute(['svn', 'commit', '-m', msg])
def branch_diff(self, args):
logging.debug("branch diff: %s" % (args))
if len(args) > 0:
branch = args[0]
else:
branch = args
if not branch.startswith("br:"):
return None
if not self._options.branch:
self._options.branch = branch
files = execute(["cm", "fbc", branch, "--format={3} {4}"],
split_lines=True)
logging.debug("got files: %s" % (files))
diff_lines = []
empty_filename = make_tempfile()
tmp_diff_from_filename = make_tempfile()
tmp_diff_to_filename = make_tempfile()
for f in files:
f = f.strip()
if not f:
continue
m = re.search(r'^(?P<branch>.*)#(?P<revno>\d+) (?P<file>.*)$', f)
if not m:
die("Could not parse 'cm fbc' response: %s" % f)
filename = m.group("file")
branch = m.group("branch")
revno = m.group("revno")
# Get the base revision with a cm find
basefiles = execute([
"cm", "find", "revs", "where", "item='" + filename + "'",
"and", "branch='" + branch + "'", "and", "revno=" + revno,
"--format={item} rev:revid:{id} "
"rev:revid:{parent}", "--nototal"
],
split_lines=True)
# We only care about the first line
m = re.search(
r'^(?P<filename>.*) '
r'(?P<revspec>rev:revid:[-\d]+) '
r'(?P<parentrevspec>rev:revid:[-\d]+)$', basefiles[0])
basefilename = m.group("filename")
newrevspec = m.group("revspec")
parentrevspec = m.group("parentrevspec")
# Cope with adds/removes
changetype = "C"
if parentrevspec == "rev:revid:-1":
changetype = "A"
elif newrevspec == "rev:revid:-1":
changetype = "R"
logging.debug(
"Type %s File %s Old %s New %s" %
(changetype, basefilename, parentrevspec, newrevspec))
old_file = new_file = empty_filename
if changetype == "A":
# File Added
self.write_file(basefilename, newrevspec, tmp_diff_to_filename)
new_file = tmp_diff_to_filename
elif changetype == "R":
# File Removed
self.write_file(basefilename, parentrevspec,
tmp_diff_from_filename)
old_file = tmp_diff_from_filename
else:
self.write_file(basefilename, parentrevspec,
tmp_diff_from_filename)
old_file = tmp_diff_from_filename
self.write_file(basefilename, newrevspec, tmp_diff_to_filename)
new_file = tmp_diff_to_filename
dl = self.diff_files(old_file, new_file, basefilename, newrevspec,
parentrevspec, changetype)
diff_lines += dl
os.unlink(empty_filename)
os.unlink(tmp_diff_from_filename)
os.unlink(tmp_diff_to_filename)
return ''.join(diff_lines)
def diff(self, revisions, files=[], extra_args=[]):
"""
Performs a diff in a Subversion repository.
If the given revision spec is empty, this will do a diff of the
modified files in the working directory. If the spec is a changelist,
it will do a diff of the modified files in that changelist. If the spec
is a single revision, it will show the changes in that revision. If the
spec is two revisions, this will do a diff between the two revisions.
SVN repositories do not support branches of branches in a way that
makes parent diffs possible, so we never return a parent diff.
"""
base = str(revisions['base'])
tip = str(revisions['tip'])
repository_info = self.get_repository_info()
diff_cmd = ['svn', 'diff', '--diff-cmd=diff', '--notice-ancestry']
changelist = None
if tip == self.REVISION_WORKING_COPY:
# Posting the working copy
diff_cmd.extend(['-r', base])
elif tip.startswith(self.REVISION_CHANGELIST_PREFIX):
# Posting a changelist
changelist = tip[len(self.REVISION_CHANGELIST_PREFIX):]
diff_cmd.extend(['--changelist', changelist])
else:
# Diff between two separate revisions. Behavior depends on whether
# or not there's a working copy
if self.options.repository_url:
# No working copy--create 'old' and 'new' URLs
if len(files) == 1:
# If there's a single file or directory passed in, we use
# that as part of the URL instead of as a separate
# filename.
repository_info.set_base_path(files[0])
files = []
new_url = (repository_info.path + repository_info.base_path +
'@' + tip)
# When the source revision is '0', assume the user wants to
# upload a diff containing all the files in 'base_path' as
# new files. If the base path within the repository is added to
# both the old and new URLs, `svn diff` will error out, since
# the base_path didn't exist at revision 0. To avoid that
# error, use the repository's root URL as the source for the
# diff.
if base == '0':
old_url = repository_info.path + '@' + base
else:
old_url = (repository_info.path +
repository_info.base_path + '@' + base)
diff_cmd.extend([old_url, new_url])
else:
# Working copy--do a normal range diff
diff_cmd.extend(['-r', '%s:%s' % (base, tip)])
diff_cmd.extend(files)
if self.history_scheduled_with_commit(changelist):
svn_show_copies_as_adds = getattr(self.options,
'svn_show_copies_as_adds', None)
if svn_show_copies_as_adds is None:
sys.stderr.write("One or more files in your changeset has "
"history scheduled with commit. Please try "
"again with '--svn-show-copies-as-adds=y/n"
"'\n")
sys.exit(1)
else:
if svn_show_copies_as_adds in 'Yy':
diff_cmd.append("--show-copies-as-adds")
diff = execute(diff_cmd, split_lines=True)
diff = self.handle_renames(diff)
diff = self.convert_to_absolute_paths(diff, repository_info)
return {
'diff': ''.join(diff),
}
def _diff_working_copy(self, base, include_files, exclude_patterns):
"""Return a diff of the working copy.
Args:
base (unicode):
The base revision to diff against.
include_files (list):
A list of file paths to include in the diff.
exclude_patterns (list):
A list of file paths to exclude from the diff.
Returns:
dict:
A dictionary containing ``diff``, ``parent_diff``, and
``base_commit_id`` keys. In the case of TFS, the parent diff key
will always be ``None``.
"""
# We pass results_unicode=False because that uses the filesystem
# encoding, but the XML results we get should always be UTF-8, and are
# well-formed with the encoding specified. We can therefore let
# ElementTree determine how to decode it.
status = self._run_tf(['vc', 'status', '/format:xml'],
results_unicode=False)
root = ET.fromstring(status)
diff = []
for pending_change in root.findall(
'./PendingSet/PendingChanges/PendingChange'):
action = pending_change.attrib['chg'].split(' ')
old_filename = \
pending_change.attrib.get('srcitem', '').encode('utf-8')
new_filename = pending_change.attrib['item'].encode('utf-8')
local_filename = pending_change.attrib['local']
old_version = \
pending_change.attrib.get('svrfm', '0').encode('utf-8')
file_type = pending_change.attrib['type']
encoding = pending_change.attrib['enc']
new_version = b'(pending)'
old_data = b''
new_data = b''
binary = (encoding == '-1')
copied = 'Branch' in action
if (not file_type or (not os.path.isfile(local_filename)
and 'Delete' not in action)):
continue
if (exclude_patterns and filename_match_any_patterns(
local_filename, exclude_patterns, base_dir=None)):
continue
if 'Add' in action:
old_filename = b'/dev/null'
if not binary:
with open(local_filename, 'rb') as f:
new_data = f.read()
old_data = b''
elif 'Delete' in action:
old_data = self._run_tf([
'vc', 'view',
'/version:%s' % old_version.decode('utf-8'),
old_filename.decode('utf-8')
],
results_unicode=False)
new_data = b''
new_version = b'(deleted)'
elif 'Edit' in action:
if not binary:
old_data = self._run_tf([
'vc', 'view',
old_filename.decode('utf-8'),
'/version:%s' % old_version.decode('utf-8')
],
results_unicode=False)
with open(local_filename, 'rb') as f:
new_data = f.read()
old_label = b'%s\t%s' % (old_filename, old_version)
new_label = b'%s\t%s' % (new_filename, new_version)
if copied:
diff.append(b'Copied from: %s\n' % old_filename)
if binary:
if 'Add' in action:
old_filename = new_filename
diff.append(b'--- %s\n' % old_label)
diff.append(b'+++ %s\n' % new_label)
diff.append(b'Binary files %s and %s differ\n' %
(old_filename, new_filename))
elif old_filename != new_filename and old_data == new_data:
# Renamed file with no changes.
diff.append(b'--- %s\n' % old_label)
diff.append(b'+++ %s\n' % new_label)
else:
old_tmp = tempfile.NamedTemporaryFile(delete=False)
old_tmp.write(old_data)
old_tmp.close()
new_tmp = tempfile.NamedTemporaryFile(delete=False)
new_tmp.write(new_data)
new_tmp.close()
unified_diff = execute([
'diff', '-u', '--label',
old_label.decode('utf-8'), '--label',
new_label.decode('utf-8'), old_tmp.name, new_tmp.name
],
extra_ignore_errors=(1, ),
log_output_on_error=False,
results_unicode=False)
diff.append(unified_diff)
os.unlink(old_tmp.name)
os.unlink(new_tmp.name)
return {
'diff': b''.join(diff),
'parent_diff': None,
'base_commit_id': base,
}
def write_file(self, filename, filespec, tmpfile):
""" Grabs a file from Plastic and writes it to a temp file """
logging.debug("Writing '%s' (rev %s) to '%s'" %
(filename, filespec, tmpfile))
execute(["cm", "cat", filespec, "--file=" + tmpfile])
def _get_current_branch(self):
"""Return the current branch of this repository."""
return execute(['hg', 'branch'], env=self._hg_env).strip()
def test_execute(self):
"""Test 'execute' method."""
self.assertTrue(re.match('.*?%d.%d.%d' % sys.version_info[:3],
process.execute([sys.executable, '-V'])))
def _changenum_diff(self, changenum):
"""
Process a diff for a particular change number. This handles both
pending and submitted changelists.
See _path_diff for the alternate version that does diffs of depot
paths.
"""
# TODO: It might be a good idea to enhance PerforceDiffParser to
# understand that newFile could include a revision tag for post-submit
# reviewing.
cl_is_pending = False
logging.info("Generating diff for changenum %s" % changenum)
description = []
if changenum == "default":
cl_is_pending = True
else:
describeCmd = ["p4"]
if self.options.p4_passwd:
describeCmd.append("-P")
describeCmd.append(self.options.p4_passwd)
describeCmd = describeCmd + ["describe", "-s", changenum]
description = execute(describeCmd, split_lines=True)
if re.search("no such changelist", description[0]):
die("CLN %s does not exist." % changenum)
# Some P4 wrappers are addding an extra line before the description
if '*pending*' in description[0] or '*pending*' in description[1]:
cl_is_pending = True
v = self.p4d_version
if cl_is_pending and (v[0] < 2002 or (v[0] == "2002" and v[1] < 2)
or changenum == "default"):
# Pre-2002.2 doesn't give file list in pending changelists,
# or we don't have a description for a default changeset,
# so we have to get it a different way.
info = execute(
["p4", "opened", "-c", str(changenum)], split_lines=True)
if (len(info) == 1 and
info[0].startswith("File(s) not opened on this client.")):
die("Couldn't find any affected files for this change.")
for line in info:
data = line.split(" ")
description.append("... %s %s" % (data[0], data[2]))
else:
# Get the file list
for line_num, line in enumerate(description):
if 'Affected files ...' in line:
break
else:
# Got to the end of all the description lines and didn't find
# what we were looking for.
die("Couldn't find any affected files for this change.")
description = description[line_num + 2:]
diff_lines = []
empty_filename = make_tempfile()
tmp_diff_from_filename = make_tempfile()
tmp_diff_to_filename = make_tempfile()
for line in description:
line = line.strip()
if not line:
continue
m = re.search(
r'\.\.\. ([^#]+)#(\d+) '
r'(add|edit|delete|integrate|branch|move/add'
r'|move/delete)', line)
if not m:
die("Unsupported line from p4 opened: %s" % line)
depot_path = m.group(1)
base_revision = int(m.group(2))
if not cl_is_pending:
# If the changelist is pending our base revision is the one
# that's currently in the depot. If we're not pending the base
# revision is actually the revision prior to this one.
base_revision -= 1
changetype = m.group(3)
logging.debug('Processing %s of %s' % (changetype, depot_path))
old_file = new_file = empty_filename
old_depot_path = new_depot_path = None
changetype_short = None
if changetype in ['edit', 'integrate']:
# A big assumption
new_revision = base_revision + 1
# We have an old file, get p4 to take this old version from the
# depot and put it into a plain old temp file for us
old_depot_path = "%s#%s" % (depot_path, base_revision)
self._write_file(old_depot_path, tmp_diff_from_filename)
old_file = tmp_diff_from_filename
# Also print out the new file into a tmpfile
if cl_is_pending:
new_file = self._depot_to_local(depot_path)
else:
new_depot_path = "%s#%s" % (depot_path, new_revision)
self._write_file(new_depot_path, tmp_diff_to_filename)
new_file = tmp_diff_to_filename
changetype_short = "M"
elif changetype in ['add', 'branch', 'move/add']:
# We have a new file, get p4 to put this new file into a pretty
# temp file for us. No old file to worry about here.
if cl_is_pending:
new_file = self._depot_to_local(depot_path)
else:
new_depot_path = "%s#%s" % (depot_path, 1)
self._write_file(new_depot_path, tmp_diff_to_filename)
new_file = tmp_diff_to_filename
changetype_short = "A"
elif changetype in ['delete', 'move/delete']:
# We've deleted a file, get p4 to put the deleted file into a
# temp file for us. The new file remains the empty file.
old_depot_path = "%s#%s" % (depot_path, base_revision)
self._write_file(old_depot_path, tmp_diff_from_filename)
old_file = tmp_diff_from_filename
changetype_short = "D"
else:
die("Unknown change type '%s' for %s" %
(changetype, depot_path))
dl = self._do_diff(old_file, new_file, depot_path, base_revision,
changetype_short)
diff_lines += dl
os.unlink(empty_filename)
os.unlink(tmp_diff_from_filename)
os.unlink(tmp_diff_to_filename)
return (''.join(diff_lines), None)
def _create_svn_repo(self):
self.svn_repo = os.path.join(self._tmpbase, 'svnrepo')
execute(['svnadmin', 'create', self.svn_repo])
def parse_revision_spec(self, revisions=[], remote=None):
"""Parse the given revision spec.
Args:
revisions (list of unicode, optional):
The 'revisions' argument is a list of revisions as specified by
the user. Items in the list do not necessarily represent a
single revision, since the user can use SCM-native syntaxes
such as "r1..r2" or "r1:r2". SCMTool-specific overrides of this
method are expected to deal with such syntaxes.
remote (unicode, optional):
This is most commonly ``origin``, but can be changed via
configuration or command-line options. This represents the
remote which is configured in Review Board.
Returns:
dict:
A dictionary with the following keys:
base (unicode):
A revision to use as the base of the resulting diff.
tip (unicode):
A revision to use as the tip of the resulting diff.
parent_base (unicode, optional):
The revision to use as the base of a parent diff.
commit_id (unicode, optional):
The ID of the single commit being posted, if not using a
range.
These will be used to generate the diffs to upload to Review Board
(or print). The diff for review will include the changes in (base,
tip], and the parent diff (if necessary) will include (parent_base,
base].
If a single revision is passed in, this will return the parent of
that revision for 'base' and the passed-in revision for 'tip'.
If zero revisions are passed in, this will return the current HEAD
as 'tip', and the upstream branch as 'base', taking into account
parent branches explicitly specified via --parent.
"""
n_revs = len(revisions)
result = {}
if not remote:
remote_branch, _ = self.get_origin()
remote = remote_branch.split('/')[0]
if n_revs == 0:
# No revisions were passed in. Start with HEAD, and find the
# tracking branch automatically.
parent_branch = self.get_parent_branch()
head_ref = self._rev_parse(self.get_head_ref())[0]
merge_base = self._rev_list_youngest_remote_ancestor(
head_ref, 'origin')
result = {
'tip': head_ref,
'commit_id': head_ref,
}
if parent_branch:
result['base'] = self._rev_parse(parent_branch)[0]
result['parent_base'] = merge_base
else:
result['base'] = merge_base
# Since the user asked us to operate on HEAD, warn them about a
# dirty working directory.
if (self.has_pending_changes() and
not self.config.get('SUPPRESS_CLIENT_WARNINGS', False)):
logging.warning('Your working directory is not clean. Any '
'changes which have not been committed '
'to a branch will not be included in your '
'review request.')
elif n_revs == 1 or n_revs == 2:
# Let `git rev-parse` sort things out.
parsed = self._rev_parse(revisions)
n_parsed_revs = len(parsed)
assert n_parsed_revs <= 3
if n_parsed_revs == 1:
# Single revision. Extract the parent of that revision to use
# as the base.
parent = self._rev_parse('%s^' % parsed[0])[0]
result = {
'base': parent,
'tip': parsed[0],
'commit_id': parsed[0],
}
elif n_parsed_revs == 2:
if parsed[1].startswith('^'):
# Passed in revisions were probably formatted as
# "base..tip". The rev-parse output includes all ancestors
# of the first part, and none of the ancestors of the
# second. Basically, the second part is the base (after
# stripping the ^ prefix) and the first is the tip.
result = {
'base': parsed[1][1:],
'tip': parsed[0],
}
else:
# First revision is base, second is tip
result = {
'base': parsed[0],
'tip': parsed[1],
}
elif n_parsed_revs == 3 and parsed[2].startswith('^'):
# Revision spec is diff-since-merge. Find the merge-base of the
# two revs to use as base.
merge_base = execute(
[self.git, 'merge-base', parsed[0], parsed[1]]).strip()
result = {
'base': merge_base,
'tip': parsed[0],
}
else:
raise InvalidRevisionSpecError(
'Unexpected result while parsing revision spec')
parent_base = self._rev_list_youngest_remote_ancestor(
result['base'], 'origin')
if parent_base != result['base']:
result['parent_base'] = parent_base
else:
raise TooManyRevisionsError
return result
def make_diff(self, merge_base, base, tip, include_files,
exclude_patterns):
"""Performs a diff on a particular branch range."""
rev_range = "%s..%s" % (base, tip)
if include_files:
include_files = ['--'] + include_files
git_cmd = [self.git]
if self._supports_git_config_flag():
git_cmd.extend(['-c', 'core.quotepath=false'])
if self.type in ('svn', 'perforce'):
diff_cmd_params = ['--no-color', '--no-prefix', '-r', '-u']
elif self.type == 'git':
diff_cmd_params = [
'--no-color', '--full-index', '--ignore-submodules'
]
if self._supports_git_config_flag():
git_cmd.extend(['-c', 'diff.noprefix=false'])
if (self.capabilities is not None
and self.capabilities.has_capability(
'diffs', 'moved_files')):
diff_cmd_params.append('-M')
else:
diff_cmd_params.append('--no-renames')
else:
assert False
# By default, don't allow using external diff commands. This prevents
# things from breaking horribly if someone configures a graphical diff
# viewer like p4merge or kaleidoscope. This can be overridden by
# setting GIT_USE_EXT_DIFF = True in ~/.reviewboardrc
if not self.config.get('GIT_USE_EXT_DIFF', False):
diff_cmd_params.append('--no-ext-diff')
diff_cmd = git_cmd + ['diff'] + diff_cmd_params
if exclude_patterns:
# If we have specified files to exclude, we will get a list of all
# changed files and run `git diff` on each un-excluded file
# individually.
changed_files_cmd = git_cmd + ['diff-tree'] + diff_cmd_params
if self.type in ('svn', 'perforce'):
# We don't want to send -u along to git diff-tree because it
# will generate diff information along with the list of
# changed files.
changed_files_cmd.remove('-u')
elif self.type == 'git':
changed_files_cmd.append('-r')
changed_files = execute(changed_files_cmd + [rev_range] +
include_files,
split_lines=True,
with_errors=False,
ignore_errors=True,
none_on_ignored_error=True,
log_output_on_error=False)
# The output of git diff-tree will be a list of entries that have
# changed between the two revisions that we give it. The last part
# of the line is the name of the file that has changed.
changed_files = remove_filenames_matching_patterns(
(filename.split()[-1] for filename in changed_files),
exclude_patterns,
base_dir=self._get_root_directory())
diff_lines = []
for filename in changed_files:
lines = execute(diff_cmd + [rev_range, '--', filename],
split_lines=True,
with_errors=False,
ignore_errors=True,
none_on_ignored_error=True,
log_output_on_error=False,
results_unicode=False)
if lines is None:
logging.error(
'Could not get diff for all files (git-diff failed '
'for "%s"). Refusing to return a partial diff.' %
filename)
diff_lines = None
break
diff_lines += lines
else:
diff_lines = execute(diff_cmd + [rev_range] + include_files,
split_lines=True,
with_errors=False,
ignore_errors=True,
none_on_ignored_error=True,
log_output_on_error=False,
results_unicode=False)
if self.type == 'svn':
return self.make_svn_diff(merge_base, diff_lines)
elif self.type == 'perforce':
return self.make_perforce_diff(merge_base, diff_lines)
else:
return b''.join(diff_lines)
def _rev_parse(self, revisions):
"""Runs `git rev-parse` and returns a list of revisions."""
if not isinstance(revisions, list):
revisions = [revisions]
return execute([self.git, 'rev-parse'] + revisions).strip().split('\n')
def test_execute(self):
"""Testing execute"""
self.assertTrue(
re.match('.*?%d.%d.%d' % sys.version_info[:3],
execute([sys.executable, '-V'])))
def get_repository_info(self):
"""Get repository information for the current Git working tree.
Returns:
rbtools.clients.RepositoryInfo:
The repository info structure.
"""
# Temporarily reset the toplevel. This is necessary for making things
# work correctly in unit tests where we may be moving the cwd around a
# lot.
self._git_toplevel = None
if not check_install(['git', '--help']):
# CreateProcess (launched via subprocess, used by check_install)
# does not automatically append .cmd for things it finds in PATH.
# If we're on Windows, and this works, save it for further use.
if (sys.platform.startswith('win')
and check_install(['git.cmd', '--help'])):
self.git = 'git.cmd'
else:
logging.debug('Unable to execute "git --help" or "git.cmd '
'--help": skipping Git')
return None
git_dir = self._execute([self.git, 'rev-parse', '--git-dir'],
ignore_errors=True).rstrip('\n')
if git_dir.startswith('fatal:') or not os.path.isdir(git_dir):
return None
# Sometimes core.bare is not set, and generates an error, so ignore
# errors. Valid values are 'true' or '1'.
bare = execute([self.git, 'config', 'core.bare'],
ignore_errors=True).strip()
self.bare = bare in ('true', '1')
# Running in directories other than the top level of
# of a work-tree would result in broken diffs on the server
if not self.bare:
git_top = execute([self.git, 'rev-parse', '--show-toplevel'],
ignore_errors=True).rstrip('\n')
# Top level might not work on old git version se we use git dir
# to find it.
if (git_top.startswith(('fatal:', 'cygdrive'))
or not os.path.isdir(git_dir)):
git_top = git_dir
self._git_toplevel = os.path.abspath(git_top)
self._head_ref = self._execute(
[self.git, 'symbolic-ref', '-q', 'HEAD'],
ignore_errors=True).strip()
# We know we have something we can work with. Let's find out
# what it is. We'll try SVN first, but only if there's a .git/svn
# directory. Otherwise, it may attempt to create one and scan
# revisions, which can be slow. Also skip SVN detection if the git
# repository was specified on command line.
git_svn_dir = os.path.join(git_dir, 'svn')
if (not getattr(self.options, 'repository_url', None)
and os.path.isdir(git_svn_dir)
and len(os.listdir(git_svn_dir)) > 0):
data = self._execute([self.git, 'svn', 'info'], ignore_errors=True)
m = re.search(r'^Repository Root: (.+)$', data, re.M)
if m:
path = m.group(1)
m = re.search(r'^URL: (.+)$', data, re.M)
if m:
base_path = m.group(1)[len(path):] or '/'
m = re.search(r'^Repository UUID: (.+)$', data, re.M)
if m:
uuid = m.group(1)
self._type = self.TYPE_GIT_SVN
m = re.search(r'Working Copy Root Path: (.+)$', data,
re.M)
if m:
local_path = m.group(1)
else:
local_path = self._git_toplevel
return SVNRepositoryInfo(path=path,
base_path=base_path,
local_path=local_path,
uuid=uuid,
supports_parent_diffs=True)
else:
# Versions of git-svn before 1.5.4 don't (appear to) support
# 'git svn info'. If we fail because of an older git install,
# here, figure out what version of git is installed and give
# the user a hint about what to do next.
version = self._execute([self.git, 'svn', '--version'],
ignore_errors=True)
version_parts = re.search('version (\d+)\.(\d+)\.(\d+)',
version)
svn_remote = self._execute(
[self.git, 'config', '--get', 'svn-remote.svn.url'],
ignore_errors=True)
if (version_parts and svn_remote and not is_valid_version(
(int(version_parts.group(1)), int(version_parts.group(2)),
int(version_parts.group(3))), (1, 5, 4))):
raise SCMError('Your installation of git-svn must be '
'upgraded to version 1.5.4 or later.')
# Okay, maybe Perforce (git-p4).
git_p4_ref = os.path.join(git_dir, 'refs', 'remotes', 'p4', 'master')
if os.path.exists(git_p4_ref):
data = self._execute([self.git, 'config', '--get', 'git-p4.port'],
ignore_errors=True)
m = re.search(r'(.+)', data)
if m:
port = m.group(1)
else:
port = os.getenv('P4PORT')
if port:
self._type = self.TYPE_GIT_P4
return RepositoryInfo(path=port,
base_path='',
local_path=self._git_toplevel,
supports_parent_diffs=True)
# Nope, it's git then.
# Check for a tracking branch and determine merge-base
self._type = self.TYPE_GIT
url = None
if getattr(self.options, 'repository_url', None):
url = self.options.repository_url
else:
upstream_branch = self._get_parent_branch()
url = self._get_origin(upstream_branch).rstrip('/')
if url.startswith('fatal:'):
raise SCMError('Could not determine remote URL for upstream '
'branch %s' % upstream_branch)
# Central bare repositories don't have origin URLs.
# We return git_dir instead and hope for the best.
if not url:
url = os.path.abspath(git_dir)
if url:
return RepositoryInfo(path=url,
base_path='',
local_path=self._git_toplevel,
supports_parent_diffs=True)
return None
def _diff_working_copy(self, base, include_files, exclude_patterns):
"""Return a diff of the working copy.
Args:
base (unicode):
The base revision to diff against.
include_files (list):
A list of file paths to include in the diff.
exclude_patterns (list):
A list of file paths to exclude from the diff.
Returns:
dict:
A dictionary containing ``diff``, ``parent_diff``, and
``base_commit_id`` keys. In the case of TFS, the parent diff key
will always be ``None``.
"""
# We pass results_unicode=False because that uses the filesystem
# encoding, but the XML results we get should always be UTF-8, and are
# well-formed with the encoding specified. We can therefore let
# ElementTree determine how to decode it.
status = self._run_tf(['status', '-format:xml'], results_unicode=False)
root = ET.fromstring(status)
diff = []
for pending_change in root.findall('./pending-changes/pending-change'):
action = pending_change.attrib['change-type'].split(', ')
new_filename = pending_change.attrib['server-item'].encode('utf-8')
local_filename = pending_change.attrib['local-item']
old_version = pending_change.attrib['version'].encode('utf-8')
file_type = pending_change.attrib.get('file-type')
new_version = b'(pending)'
old_data = b''
new_data = b''
copied = 'branch' in action
if (not file_type or (not os.path.isfile(local_filename)
and 'delete' not in action)):
continue
if (exclude_patterns and filename_match_any_patterns(
local_filename, exclude_patterns, base_dir=None)):
continue
if 'rename' in action:
old_filename = \
pending_change.attrib['source-item'].encode('utf-8')
else:
old_filename = new_filename
if copied:
old_filename = \
pending_change.attrib['source-item'].encode('utf-8')
old_version = ('%d' % self._convert_symbolic_revision(
'W', old_filename.decode('utf-8')))
if 'add' in action:
old_filename = b'/dev/null'
if file_type != 'binary':
with open(local_filename) as f:
new_data = f.read()
old_data = b''
elif 'delete' in action:
old_data = self._run_tf([
'print',
'-version:%s' % old_version.decode('utf-8'),
old_filename.decode('utf-8')
],
results_unicode=False)
new_data = b''
new_version = b'(deleted)'
elif 'edit' in action:
old_data = self._run_tf([
'print',
'-version:%s' % old_version.decode('utf-8'),
old_filename.decode('utf-8')
],
results_unicode=False)
with open(local_filename) as f:
new_data = f.read()
old_label = b'%s\t%s' % (old_filename, old_version)
new_label = b'%s\t%s' % (new_filename, new_version)
if copied:
diff.append(b'Copied from: %s\n' % old_filename)
if file_type == 'binary':
if 'add' in action:
old_filename = new_filename
diff.append(b'--- %s\n' % old_label)
diff.append(b'+++ %s\n' % new_label)
diff.append(b'Binary files %s and %s differ\n' %
(old_filename, new_filename))
elif old_filename != new_filename and old_data == new_data:
# Renamed file with no changes
diff.append(b'--- %s\n' % old_label)
diff.append(b'+++ %s\n' % new_label)
else:
old_tmp = tempfile.NamedTemporaryFile(delete=False)
old_tmp.write(old_data)
old_tmp.close()
new_tmp = tempfile.NamedTemporaryFile(delete=False)
new_tmp.write(new_data)
new_tmp.close()
unified_diff = execute([
'diff', '-u', '--label',
old_label.decode('utf-8'), '--label',
new_label.decode('utf-8'), old_tmp.name, new_tmp.name
],
extra_ignore_errors=(1, ),
log_output_on_error=False,
results_unicode=False)
diff.append(unified_diff)
os.unlink(old_tmp.name)
os.unlink(new_tmp.name)
if len(root.findall('./candidate-pending-changes/pending-change')) > 0:
logging.warning('There are added or deleted files which have not '
'been added to TFS. These will not be included '
'in your review request.')
return {
'diff': b''.join(diff),
'parent_diff': None,
'base_commit_id': base,
}
def get_commit_history(self, revisions):
"""Return the commit history specified by the revisions.
Args:
revisions (dict):
A dictionary of revisions to generate history for, as returned
by :py:meth:`parse_revision_spec`.
Returns:
list of dict:
The list of history entries, in order. The dictionaries have the
following keys:
``commit_id``:
The unique identifier of the commit.
``parent_id``:
The unique identifier of the parent commit.
``author_name``:
The name of the commit's author.
``author_email``:
The e-mail address of the commit's author.
``author_date``:
The date the commit was authored.
``committer_name``:
The committer's name.
``committer_email``:
The e-mail address of the committer.
``committer_date``:
The date the commit was committed.
``commit_message``:
The commit's message.
Raises:
rbtools.clients.errors.SCMError:
The history is non-linear or there is a commit with no parents.
"""
log_fields = {
'commit_id': b'%H',
'parent_id': b'%P',
'author_name': b'%an',
'author_email': b'%ae',
'author_date': b'%ad',
'committer_name': b'%cn',
'committer_email': b'%ce',
'committer_date': b'%cd',
'commit_message': b'%B',
}
# 0x1f is the ASCII field separator. It is a non-printable character
# that should not appear in any field in `git log`.
log_format = b'%x1f'.join(six.itervalues(log_fields))
log_entries = execute([
self.git,
b'log',
b'-z',
b'--reverse',
b'--pretty=format:%s' % log_format,
b'--date=iso8601-strict',
b'%s..%s' % (bytes(revisions['base']), bytes(revisions['tip'])),
],
ignore_errors=True,
none_on_ignored_error=True,
results_unicode=True)
if not log_entries:
return None
history = []
field_names = six.viewkeys(log_fields)
for log_entry in log_entries.split(self._NUL):
fields = log_entry.split(self._FIELD_SEP)
entry = dict(zip(field_names, fields))
parents = entry['parent_id'].split()
if len(parents) > 1:
raise SCMError(
'The Git SCMClient only supports posting commit histories '
'that are entirely linear.')
elif len(parents) == 0:
raise SCMError(
'The Git SCMClient only supports posting commits that '
'have exactly one parent.')
history.append(entry)
return history
def changenum_diff(self, changenum):
logging.debug("changenum_diff: %s" % (changenum))
files = execute([
"cm", "log", "cs:" + changenum, "--csFormat={items}",
"--itemFormat={shortstatus} {path} "
"rev:revid:{revid} rev:revid:{parentrevid} "
"src:{srccmpath} rev:revid:{srcdirrevid} "
"dst:{dstcmpath} rev:revid:{dstdirrevid}{newline}"
],
split_lines=True)
logging.debug("got files: %s" % (files))
# Diff generation based on perforce client
diff_lines = []
empty_filename = make_tempfile()
tmp_diff_from_filename = make_tempfile()
tmp_diff_to_filename = make_tempfile()
for f in files:
f = f.strip()
if not f:
continue
m = re.search(
r'(?P<type>[ACIMR]) (?P<file>.*) '
r'(?P<revspec>rev:revid:[-\d]+) '
r'(?P<parentrevspec>rev:revid:[-\d]+) '
r'src:(?P<srcpath>.*) '
r'(?P<srcrevspec>rev:revid:[-\d]+) '
r'dst:(?P<dstpath>.*) '
r'(?P<dstrevspec>rev:revid:[-\d]+)$', f)
if not m:
die("Could not parse 'cm log' response: %s" % f)
changetype = m.group("type")
filename = m.group("file")
if changetype == "M":
# Handle moved files as a delete followed by an add.
# Clunky, but at least it works
oldfilename = m.group("srcpath")
oldspec = m.group("srcrevspec")
newfilename = m.group("dstpath")
newspec = m.group("dstrevspec")
self.write_file(oldfilename, oldspec, tmp_diff_from_filename)
dl = self.diff_files(tmp_diff_from_filename, empty_filename,
oldfilename, "rev:revid:-1", oldspec,
changetype)
diff_lines += dl
self.write_file(newfilename, newspec, tmp_diff_to_filename)
dl = self.diff_files(empty_filename, tmp_diff_to_filename,
newfilename, newspec, "rev:revid:-1",
changetype)
diff_lines += dl
else:
newrevspec = m.group("revspec")
parentrevspec = m.group("parentrevspec")
logging.debug(
"Type %s File %s Old %s New %s" %
(changetype, filename, parentrevspec, newrevspec))
old_file = new_file = empty_filename
if (changetype in ['A']
or (changetype in ['C', 'I']
and parentrevspec == "rev:revid:-1")):
# File was Added, or a Change or Merge (type I) and there
# is no parent revision
self.write_file(filename, newrevspec, tmp_diff_to_filename)
new_file = tmp_diff_to_filename
elif changetype in ['C', 'I']:
# File was Changed or Merged (type I)
self.write_file(filename, parentrevspec,
tmp_diff_from_filename)
old_file = tmp_diff_from_filename
self.write_file(filename, newrevspec, tmp_diff_to_filename)
new_file = tmp_diff_to_filename
elif changetype in ['R']:
# File was Removed
self.write_file(filename, parentrevspec,
tmp_diff_from_filename)
old_file = tmp_diff_from_filename
else:
die("Don't know how to handle change type '%s' for %s" %
(changetype, filename))
dl = self.diff_files(old_file, new_file, filename, newrevspec,
parentrevspec, changetype)
diff_lines += dl
os.unlink(empty_filename)
os.unlink(tmp_diff_from_filename)
os.unlink(tmp_diff_to_filename)
return ''.join(diff_lines)
def _run_svn(self, command):
return execute(['svn'] + command,
env=None,
split_lines=False,
ignore_errors=False,
extra_ignore_errors=())
def diff_files(self,
old_file,
new_file,
filename,
newrevspec,
parentrevspec,
changetype,
ignore_unmodified=False):
"""
Do the work of producing a diff for Plastic (based on the Perforce one)
old_file - The absolute path to the "old" file.
new_file - The absolute path to the "new" file.
filename - The file in the Plastic workspace
newrevspec - The revid spec of the changed file
parentrevspecspec - The revision spec of the "old" file
changetype - The change type as a single character string
ignore_unmodified - If true, will return an empty list if the file
is not changed.
Returns a list of strings of diff lines.
"""
if filename.startswith(self.workspacedir):
filename = filename[len(self.workspacedir):]
diff_cmd = ["diff", "-urN", old_file, new_file]
# Diff returns "1" if differences were found.
dl = execute(diff_cmd,
extra_ignore_errors=(1, 2),
translate_newlines=False)
# If the input file has ^M characters at end of line, lets ignore them.
dl = dl.replace('\r\r\n', '\r\n')
dl = dl.splitlines(True)
# Special handling for the output of the diff tool on binary files:
# diff outputs "Files a and b differ"
# and the code below expects the output to start with
# "Binary files "
if (len(dl) == 1 and dl[0].startswith('Files %s and %s differ' %
(old_file, new_file))):
dl = ['Binary files %s and %s differ\n' % (old_file, new_file)]
if dl == [] or dl[0].startswith("Binary files "):
if dl == []:
if ignore_unmodified:
return []
else:
print "Warning: %s in your changeset is unmodified" % \
filename
dl.insert(
0,
"==== %s (%s) ==%s==\n" % (filename, newrevspec, changetype))
dl.append('\n')
else:
dl[0] = "--- %s\t%s\n" % (filename, parentrevspec)
dl[1] = "+++ %s\t%s\n" % (filename, newrevspec)
# Not everybody has files that end in a newline. This ensures
# that the resulting diff file isn't broken.
if dl[-1][-1] != '\n':
dl.append('\n')
return dl
def get_current_branch(self):
"""Returns the name of the current branch."""
return execute([self.git, "rev-parse", "--abbrev-ref", "HEAD"],
ignore_errors=True).strip()
def parse_revision_spec(self, revisions=[]):
"""Parses the given revision spec.
The 'revisions' argument is a list of revisions as specified by the
user. Items in the list do not necessarily represent a single revision,
since the user can use SCM-native syntaxes such as "r1..r2" or "r1:r2".
SCMTool-specific overrides of this method are expected to deal with
such syntaxes.
This will return a dictionary with the following keys:
'base': A revision to use as the base of the resulting diff.
'tip': A revision to use as the tip of the resulting diff.
These will be used to generate the diffs to upload to Review Board (or
print). The diff for review will include the changes in (base, tip].
If a single revision is passed in, this will return the parent of that
revision for 'base' and the passed-in revision for 'tip'.
If zero revisions are passed in, this will return the most recently
checked-out revision for 'base' and a special string indicating the
working copy for 'tip'.
The SVN SCMClient never fills in the 'parent_base' key. Users who are
using other patch-stack tools who want to use parent diffs with SVN
will have to generate their diffs by hand.
"""
n_revisions = len(revisions)
if n_revisions == 1 and ':' in revisions[0]:
revisions = revisions[0].split(':')
n_revisions = len(revisions)
if n_revisions == 0:
# Most recent checked-out revision -- working copy
# TODO: this should warn about mixed-revision working copies that
# affect the list of files changed (see bug 2392).
return {
'base': 'BASE',
'tip': self.REVISION_WORKING_COPY,
}
elif n_revisions == 1:
# Either a numeric revision (n-1:n) or a changelist
revision = revisions[0]
try:
revision = self._convert_symbolic_revision(revision)
return {
'base': revision - 1,
'tip': revision,
}
except ValueError:
# It's not a revision--let's try a changelist. This only makes
# sense if we have a working copy.
if not self.options.repository_url:
status = execute([
'svn', 'status', '--cl',
str(revision), '--ignore-externals', '--xml'
])
cl = ElementTree.fromstring(status).find('changelist')
if cl is not None:
# TODO: this should warn about mixed-revision working
# copies that affect the list of files changed (see
# bug 2392).
return {
'base': 'BASE',
'tip': self.REVISION_CHANGELIST_PREFIX + revision
}
raise InvalidRevisionSpecError(
'"%s" does not appear to be a valid revision or '
'changelist name' % revision)
elif n_revisions == 2:
# Diff between two numeric revisions
try:
return {
'base': self._convert_symbolic_revision(revisions[0]),
'tip': self._convert_symbolic_revision(revisions[1]),
}
except ValueError:
raise InvalidRevisionSpecError(
'Could not parse specified revisions: %s' % revisions)
else:
raise TooManyRevisionsError
def _get_merge_base(self, rev1, rev2):
"""Returns the merge base."""
return execute([self.git, "merge-base", rev1, rev2]).strip()
def get_url_prop(path):
url = execute(["svn", "propget", "reviewboard:url", path],
with_errors=False).strip()
return url or None
def _execute(self, cmd, *args, **kwargs):
"""
Prints the results of the executed command and returns
the data result from execute.
"""
return execute(cmd, ignore_errors=True, *args, **kwargs)
def get_repository_info(self):
if not check_install('git --help'):
# CreateProcess (launched via subprocess, used by check_install)
# does not automatically append .cmd for things it finds in PATH.
# If we're on Windows, and this works, save it for further use.
if (sys.platform.startswith('win')
and check_install('git.cmd --help')):
self.git = 'git.cmd'
else:
return None
git_dir = execute([self.git, "rev-parse", "--git-dir"],
ignore_errors=True).rstrip("\n")
if git_dir.startswith("fatal:") or not os.path.isdir(git_dir):
return None
self.bare = execute([self.git, "config",
"core.bare"]).strip() == 'true'
# post-review in directories other than the top level of
# of a work-tree would result in broken diffs on the server
if not self.bare:
git_top = execute([self.git, "rev-parse", "--show-toplevel"],
ignore_errors=True).rstrip("\n")
# Top level might not work on old git version se we use git dir
# to find it.
if git_top.startswith("fatal:") or not os.path.isdir(git_dir):
git_top = git_dir
os.chdir(os.path.abspath(git_top))
self.head_ref = execute([self.git, 'symbolic-ref', '-q', 'HEAD'],
ignore_errors=True).strip()
# We know we have something we can work with. Let's find out
# what it is. We'll try SVN first, but only if there's a .git/svn
# directory. Otherwise, it may attempt to create one and scan
# revisions, which can be slow. Also skip SVN detection if the git
# repository was specified on command line.
git_svn_dir = os.path.join(git_dir, 'svn')
if (not self.options.repository_url and os.path.isdir(git_svn_dir)
and len(os.listdir(git_svn_dir)) > 0):
data = execute([self.git, "svn", "info"], ignore_errors=True)
m = re.search(r'^Repository Root: (.+)$', data, re.M)
if m:
path = m.group(1)
m = re.search(r'^URL: (.+)$', data, re.M)
if m:
base_path = m.group(1)[len(path):] or "/"
m = re.search(r'^Repository UUID: (.+)$', data, re.M)
if m:
uuid = m.group(1)
self.type = "svn"
# Get SVN tracking branch
if self.options.parent_branch:
self.upstream_branch = self.options.parent_branch
else:
data = execute([self.git, "svn", "rebase", "-n"],
ignore_errors=True)
m = re.search(r'^Remote Branch:\s*(.+)$', data,
re.M)
if m:
self.upstream_branch = m.group(1)
else:
sys.stderr.write('Failed to determine SVN '
'tracking branch. Defaulting'
'to "master"\n')
self.upstream_branch = 'master'
return SVNRepositoryInfo(path=path,
base_path=base_path,
uuid=uuid,
supports_parent_diffs=True)
else:
# Versions of git-svn before 1.5.4 don't (appear to) support
# 'git svn info'. If we fail because of an older git install,
# here, figure out what version of git is installed and give
# the user a hint about what to do next.
version = execute([self.git, "svn", "--version"],
ignore_errors=True)
version_parts = re.search('version (\d+)\.(\d+)\.(\d+)',
version)
svn_remote = execute(
[self.git, "config", "--get", "svn-remote.svn.url"],
ignore_errors=True)
if (version_parts and not self.is_valid_version(
(int(version_parts.group(1)), int(version_parts.group(2)),
int(version_parts.group(3))), (1, 5, 4)) and svn_remote):
die("Your installation of git-svn must be upgraded to "
"version 1.5.4 or later")
# Okay, maybe Perforce.
# TODO
# Nope, it's git then.
# Check for a tracking branch and determine merge-base
self.upstream_branch = ''
if self.head_ref:
short_head = self._strip_heads_prefix(self.head_ref)
merge = execute(
[self.git, 'config', '--get',
'branch.%s.merge' % short_head],
ignore_errors=True).strip()
remote = execute(
[self.git, 'config', '--get',
'branch.%s.remote' % short_head],
ignore_errors=True).strip()
merge = self._strip_heads_prefix(merge)
if remote and remote != '.' and merge:
self.upstream_branch = '%s/%s' % (remote, merge)
url = None
if self.options.repository_url:
url = self.options.repository_url
self.upstream_branch = self.get_origin(self.upstream_branch,
True)[0]
else:
self.upstream_branch, origin_url = \
self.get_origin(self.upstream_branch, True)
if not origin_url or origin_url.startswith("fatal:"):
self.upstream_branch, origin_url = self.get_origin()
url = origin_url.rstrip('/')
# Central bare repositories don't have origin URLs.
# We return git_dir instead and hope for the best.
if not url:
url = os.path.abspath(git_dir)
# There is no remote, so skip this part of upstream_branch.
self.upstream_branch = self.upstream_branch.split('/')[-1]
if url:
self.type = "git"
return RepositoryInfo(path=url,
base_path='',
supports_parent_diffs=True)
return None
def main(self, *args):
"""Create and update review requests."""
# The 'args' tuple must be made into a list for some of the
# SCM Clients code. The way arguments were structured in
# post-review meant this was a list, and certain parts of
# the code base try and concatenate args to the end of
# other lists. Until the client code is restructured and
# cleaned up we will satisfy the assumption here.
self.cmd_args = list(args)
self.post_process_options()
origcwd = os.path.abspath(os.getcwd())
repository_info, self.tool = self.initialize_scm_tool(
client_name=self.options.repository_type)
server_url = self.get_server_url(repository_info, self.tool)
api_client, api_root = self.get_api(server_url)
self.setup_tool(self.tool, api_root=api_root)
if (self.options.exclude_patterns
and not self.tool.supports_diff_exclude_patterns):
raise CommandError(
'The %s backend does not support excluding files via the '
'-X/--exclude commandline options or the EXCLUDE_PATTERNS '
'.reviewboardrc option.' % self.tool.name)
# Check if repository info on reviewboard server match local ones.
repository_info = repository_info.find_server_repository_info(api_root)
if self.options.diff_filename:
self.revisions = None
parent_diff = None
base_commit_id = None
commit_id = None
if self.options.diff_filename == '-':
if hasattr(sys.stdin, 'buffer'):
# Make sure we get bytes on Python 3.x
diff = sys.stdin.buffer.read()
else:
diff = sys.stdin.read()
else:
try:
diff_path = os.path.join(origcwd,
self.options.diff_filename)
with open(diff_path, 'rb') as fp:
diff = fp.read()
except IOError as e:
raise CommandError('Unable to open diff filename: %s' % e)
else:
self.revisions = get_revisions(self.tool, self.cmd_args)
if self.revisions:
extra_args = None
else:
extra_args = self.cmd_args
# Generate a diff against the revisions or arguments, filtering
# by the requested files if provided.
diff_info = self.tool.diff(
revisions=self.revisions,
include_files=self.options.include_files or [],
exclude_patterns=self.options.exclude_patterns or [],
extra_args=extra_args)
diff = diff_info['diff']
parent_diff = diff_info.get('parent_diff')
base_commit_id = diff_info.get('base_commit_id')
commit_id = diff_info.get('commit_id')
logging.debug('Generated diff size: %d bytes', len(diff))
if parent_diff:
logging.debug('Generated parent diff size: %d bytes',
len(parent_diff))
repository = (self.options.repository_name
or self.options.repository_url
or self.get_repository_path(repository_info, api_root))
base_dir = self.options.basedir or repository_info.base_path
if repository is None:
raise CommandError('Could not find the repository on the Review '
'Board server.')
if len(diff) == 0:
raise CommandError("There don't seem to be any diffs!")
# Validate the diffs to ensure that they can be parsed and that
# all referenced files can be found.
#
# Review Board 2.0.14+ (with the diffs.validation.base_commit_ids
# capability) is required to successfully validate against hosting
# services that need a base_commit_id. This is basically due to
# the limitations of a couple Git-specific hosting services
# (Beanstalk, Bitbucket, and Unfuddle).
#
# In order to validate, we need to either not be dealing with a
# base commit ID (--diff-filename), or be on a new enough version
# of Review Board, or be using a non-Git repository.
can_validate_base_commit_ids = \
self.tool.capabilities.has_capability('diffs', 'validation',
'base_commit_ids')
if (not base_commit_id or can_validate_base_commit_ids
or self.tool.name != 'Git'):
# We can safely validate this diff before posting it, but we
# need to ensure we only pass base_commit_id if the capability
# is set.
validate_kwargs = {}
if can_validate_base_commit_ids:
validate_kwargs['base_commit_id'] = base_commit_id
try:
diff_validator = api_root.get_diff_validation()
diff_validator.validate_diff(repository,
diff,
parent_diff=parent_diff,
base_dir=base_dir,
**validate_kwargs)
except APIError as e:
msg_prefix = ''
if e.error_code == 207:
msg_prefix = '%s: ' % e.rsp['file']
raise CommandError('Error validating diff\n\n%s%s' %
(msg_prefix, e))
except AttributeError:
# The server doesn't have a diff validation resource. Post as
# normal.
pass
if (repository_info.supports_changesets
and not self.options.diff_filename
and 'changenum' in diff_info):
changenum = diff_info['changenum']
else:
changenum = self.tool.get_changenum(self.revisions)
# Not all scm clients support get_changenum, so if get_changenum
# returns None (the default for clients that don't have changenums),
# we'll prefer the existing commit_id.
commit_id = changenum or commit_id
if self.options.update and self.revisions:
review_request = guess_existing_review_request(
repository_info,
self.options.repository_name,
api_root,
api_client,
self.tool,
self.revisions,
guess_summary=False,
guess_description=False,
is_fuzzy_match_func=self._ask_review_request_match,
submit_as=self.options.submit_as)
if not review_request or not review_request.id:
raise CommandError('Could not determine the existing review '
'request to update.')
self.options.rid = review_request.id
# If only certain files within a commit are being submitted for review,
# do not include the commit id. This prevents conflicts if multiple
# files from the same commit are posted for review separately.
if self.options.include_files or self.options.exclude_patterns:
commit_id = None
request_id, review_url = self.post_request(
repository_info,
repository,
server_url,
api_root,
self.options.rid,
changenum=changenum,
diff_content=diff,
parent_diff_content=parent_diff,
commit_id=commit_id,
base_commit_id=base_commit_id,
submit_as=self.options.submit_as,
base_dir=base_dir)
diff_review_url = review_url + 'diff/'
print('Review request #%s posted.' % request_id)
print()
print(review_url)
print(diff_review_url)
# Load the review up in the browser if requested to.
if self.options.open_browser:
if sys.platform == 'darwin':
# The 'webbrowser' module currently does a bunch of stuff with
# AppleScript, which is broken on macOS 10.12.5. See
# https://bugs.python.org/issue30392 for more discussion.
try:
execute(['open', review_url])
except Exception as e:
logging.exception('Error opening review URL %s: %s',
review_url, e)
else:
try:
import webbrowser
if 'open_new_tab' in dir(webbrowser):
# open_new_tab is only in python 2.5+
webbrowser.open_new_tab(review_url)
elif 'open_new' in dir(webbrowser):
webbrowser.open_new(review_url)
else:
os.system('start %s' % review_url)
except Exception as e:
logging.exception('Error opening review URL %s: %s',
review_url, e)
def get_raw_commit_message(self, revisions):
"""Extracts the commit message based on the provided revision range."""
return execute(
[self.git, 'log', '--reverse', '--pretty=format:%s%n%n%b',
'^%s' % revisions['base'], revisions['tip']],
ignore_errors=True).strip()
def get_commit_history(self, revisions):
"""Return the commit history specified by the revisions.
Args:
revisions (dict):
A dictionary of revisions to generate history for, as returned
by :py:meth:`parse_revision_spec`.
Returns:
list of dict:
This list of history entries, in order.
Raises:
rbtools.clients.errors.SCMError:
The history is non-linear or there is a commit with no parents.
"""
log_fields = {
'commit_id': b'{node}',
'parent_id': b'{p1node}',
'author_name': b'{author|person}',
'author_email': b'{author|email}',
'author_date': b'{date|rfc3339date}',
'parent2': b'{p2node}',
'commit_message': b'{desc}',
}
log_format = self._FIELD_SEP_ESC.join(six.itervalues(log_fields))
log_entries = execute([
b'hg',
b'log',
b'--template',
br'%s%s' % (log_format, self._RECORD_SEP_ESC),
b'-r',
b'%(base)s::%(tip)s and not %(base)s' % revisions,
],
ignore_errors=True,
none_on_ignored_error=True,
results_unicode=True)
if not log_entries:
return None
history = []
field_names = six.viewkeys(log_fields)
# The ASCII record separator will be appended to every record, so if we
# attempt to split the entire output by the record separator, we will
# end up with an empty ``log_entry`` at the end, which will cause
# errors.
for log_entry in log_entries[:-1].split(self._RECORD_SEP):
fields = log_entry.split(self._FIELD_SEP)
entry = dict(zip(field_names, fields))
# We do not want `parent2` to be included in the entry because
# the entry's items are used as the keyword arguments to the
# method that uploads a commit and it would be unexpected.
if entry.pop('parent2') != self.NO_PARENT:
raise SCMError(
'The Mercurial SCMClient only supports posting commit '
'histories that are entirely linear.')
elif entry['parent_id'] == self.NO_PARENT:
raise SCMError(
'The Mercurial SCMClient only supports posting commits '
'that have exactly one parent.')
history.append(entry)
return history