def parse_patchfile(patchfile, block_dict):
"""Parse the patchfile and update corresponding block ranges."""
# https://pypi.python.org/pypi/whatthepatch/0.0.2
diffs = None
with open(patchfile) as stream:
diffs = whatthepatch.parse_patch(stream.read())
for diff in diffs:
curr_file = diff.header.old_path
# In some cases, whatthepatch will set the path to "a/path/to/file"
# instead of "path/to/file", so we strip "a/" away to match with
# the dictionary key in block_dict
if curr_file.startswith("a/"):
curr_file = curr_file[2:]
# change format: [line before patch, line after patch, text]
for change in diff.changes:
# line removed
if change[0] and not change[1]:
curr_line = int(change[0])
blocks = block_dict.get(curr_file, [])
blocks = Block.update_block_ranges(blocks, curr_line, -1)
block_dict[curr_file] = blocks
# line added
elif not change[0] and change[1]:
curr_line = int(change[1])
blocks = block_dict.get(curr_file, [])
blocks = Block.update_block_ranges(blocks, curr_line, 1)
block_dict[curr_file] = blocks
# Set ranges to updated ranges
for curr_file in block_dict:
for block in block_dict[curr_file]:
block.range = block.new_range
return block_dict
def get_diff_patches(commit: Commit,
split_per_method=True,
only_unstaged=False) -> List[str]:
'''
Creates a patch file containing all the diffs in the repository and then
returns all those patches as a list of patches
'''
# this command will output the diff information into stdout
command = 'git --no-pager diff'
if commit and not only_unstaged:
command += ' {hash}~ {hash}'.format(hash=commit.commit_id)
diff_data, _ = run_cmd(commit.repository, command)
# TODO: MY GOODNESS this is such a hack
if not diff_data:
command = 'git --no-pager diff'
diff_data, _ = run_cmd(commit.repository, command)
# TODO: GETTING EVEN WORSE. This is when there is a commit already created
if not diff_data and not only_unstaged:
command = 'git diff origin/bug_buddy..HEAD'
diff_data, _ = run_cmd(commit.repository, command)
raw_patches = diff_data.split('diff --git ')[1:]
# covert the list of patches into whatthepatch patch objects
patches = [
list(whatthepatch.parse_patch(patch))[0] for patch in raw_patches
]
# if split_per_method and commit.function_histories:
# patches = _split_patches_by_method(commit, patches)
return patches
def analyze_patch(patch, check_annotations):
files = get_files(patch)
changed = set(files['touched']) | set(files['moved'].keys())
if check_annotations:
newed = set(files['added']) | set(files['deleted'])
info = defaultdict(lambda: [])
for diff in whatthepatch.parse_patch(patch):
h = diff.header
if not h:
continue
old_p = h.old_path
old_p = old_p[2:] if old_p.startswith('a/') else old_p
if old_p in newed:
# the file has just been added or deleted,
# so nothing to compute
continue
for old, new, _ in diff.changes:
if old is not None and new is None:
# removed line
info[old_p].append(old)
files = list(info.keys())
if files:
annotations = Annotate.get(files, node='tip')
stats = analyze_annotations(info, annotations)
return stats, changed
return {'deleted': {}, 'all': {}}, changed
def analyze_diff(self, diff):
def status(change):
if change[0] is None:
return 'insert'
elif change[1] is None:
return 'delete'
else:
return 'equal'
out = []
for hunk in whatthepatch.parse_patch(diff.decode('utf-8')):
file_changes = []
in_changeset = False
for mode in [('insert', 1), ('delete', 0)]:
for change in hunk.changes:
if status(change) == mode[0] and not in_changeset:
in_changeset = True
file_changes.append({
'start': change[mode[1]],
'type': mode[0]
})
elif status(change) == 'equal' and in_changeset:
in_changeset = False
file_changes[-1]['end'] = change[mode[1]] - 1
out.append({'header': hunk.header, 'changes': file_changes})
return out
def parse_changes(repo, changes):
for change in changes:
for diff in wtp.parse_patch(get_diff(repo, change)):
removed = list()
added = list()
if not (diff.header.old_path.endswith(".java")
or diff.header.new_path.endswith(".java")):
continue
for r, a, text in diff.changes:
if r:
removed.append(r)
if a:
added.append(a)
mremoved_blocks = set()
cremoved_blocks = set()
if diff.header.old_path != "/dev/null":
logger.info("Generating XML for file %s @ %s", change.old.path,
change.old.sha)
ftext = repo[change.old.sha].as_raw_string()
mremoved_blocks, cremoved_blocks = get_blocks(ftext, removed)
madded_blocks = set()
cadded_blocks = set()
if diff.header.new_path != "/dev/null":
logger.info("Generating XML for file %s @ %s", change.new.path,
change.new.sha)
ftext = repo[change.new.sha].as_raw_string()
madded_blocks, cadded_blocks = get_blocks(ftext, added)
yield mremoved_blocks, madded_blocks, cremoved_blocks, cadded_blocks
def generate(patch):
'''
This function generates ...
'''
mapping = {}
download_mapping()
paths = []
for diff in whatthepatch.parse_patch(patch.decode('utf-8')):
# Get old and new path, for files that have been renamed.
path = diff.header.new_path[2:] if diff.header.new_path.startswith(
'b/') else diff.header.new_path
# If the diff doesn't contain any changes, we skip it.
if diff.changes is None:
continue
# If the file is not a source file, we skip it (as we already know
# we have no coverage information for it).
if not coverage_supported(path):
continue
paths.append(path)
with sqlite3.connect('chunk_mapping.db') as conn:
c = conn.cursor()
for path in paths:
c.execute('SELECT chunk FROM files WHERE path=?',
(path.encode('utf-8'), ))
mapping[path] = c.fetchall()
return mapping
def diff(repo):
repopath = posixpath.join(workspace, repo)
branch = request.args.get('branch', '')
checkoutBranch(branch, repopath)
if request.method == 'GET':
graph1 = request.args.get('a', '')
graph2 = request.args.get('b', '')
fileGraph1 = FileGraph(graph1, domain, repopath, repo)
fileGraph2 = FileGraph(graph2, domain, repopath, repo)
log.info(fileGraph1.filename)
log.info(fileGraph2.filename)
patchtext = diffnoindex(fileGraph1.filename, fileGraph2.filename,
repopath)
# WTF tuple-structure in wthatthepatch
# https://pypi.org/project/whatthepatch/
patch = list(whatthepatch.parse_patch(patchtext))[0][1]
log.info(patchtext)
for diff in patch:
a, b, line = diff
if (b == None):
print("In a, not in b: " + line)
if (a == None):
print("In b, not in a: " + line)
#TODO: Do something with this ... huh?
return patchtext
def test_git_oneline_change(self):
with open("tests/casefiles/git-oneline-change.diff") as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(
header=headerobj(
index_path=None,
old_path="oneline.txt",
old_version="f56f98d",
new_path="oneline.txt",
new_version="169ceeb",
),
changes=[
(1, None, "Adding a one-line file."),
(None, 1, "Changed a one-line file."),
],
text="\n".join(lines[:34]) + "\n",
)
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def build_changelog(change: File) -> List[str]:
lines = ['# Changelogs']
patch = whatthepatch.parse_patch(change.patch).__next__()
oldversions: Dict[str, str] = {}
newversions: Dict[str, str] = {}
for p in patch.changes:
if len(p) > 1:
if p[1] is None:
req = Requirement.parse(p[2])
oldversions[req.name] = req.specs[0][1]
elif p[0] is None:
req = Requirement.parse(p[2])
newversions[req.name] = req.specs[0][1]
else:
lines.append(f'> Warning: {p[0]} is pinned to a specific version.')
for package in newversions:
old = packaging.version.parse(oldversions.get(package))
new = packaging.version.parse(newversions.get(package))
changes = changelogs.get(package)
logged = False
for version_string in changes.keys():
v = packaging.version.parse(version_string)
if old < v <= new:
lines.append(f'## {package} {version_string}')
lines.append(changes[version_string])
logged = True
if not logged:
lines.append(f'## {package} {new}')
lines.append('No release notes found.')
return lines
def parse_diff(diff, reduce=False):
if not diff or not diff.startswith("@@"):
return []
# split hunks
parts = []
for l in diff.split("\n"):
if l.startswith("@@"):
parts.append([])
parts[-1].append(l)
results = []
position = 0
if reduce:
parts = parts[-1:]
# parse each hunk
for part in parts:
diff = whatthepatch.parse_patch(part).next() # only one file = only one diff
result = [["comment", u"…", u"…", part[0], position]]
for old, new, text in diff.changes:
position += 1
mode = " " if old and new else "-" if old else "+"
result.append([DIFF_LINE_TYPES[mode], old or "", new or "", mode + text, position])
position += 1
if reduce:
result = result[0:1] + result[1:][-12:]
results.append(result)
return chain.from_iterable(results)
def test_git_oneline_change(self):
with open('tests/casefiles/git-oneline-change.diff') as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(
header=headerobj(
index_path=None,
old_path='oneline.txt',
old_version='f56f98d',
new_path='oneline.txt',
new_version='169ceeb'
),
changes=[
(1, None, 'Adding a one-line file.'),
(None, 1, 'Changed a one-line file.')
],
text='\n'.join(lines[:34]) + '\n'
)
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_diff_rcs(self):
with open('tests/casefiles/diff-rcs.diff') as f:
diff_text = f.read()
diff = next(wtp.parse_patch(diff_text))
new_text = wtp.apply.apply_diff(diff, self.lao)
self.assertEqual(new_text, self.tzu)
def handle_diffs(diff_src, single_desc, multi_desc):
with open(diff_src) as f:
text = f.read()
diff = [x for x in whatthepatch.parse_patch(text)]
for i in diff:
rr = extract_lines(i.changes)
for j in rr:
dump_data(j, single_desc, multi_desc)
def _find_changed_files(self, current_id, new_id):
"""
Calculates the set of _changed_ files!
"""
# Grab the raw git diff
diff_text = self.repo.git.diff(current_id, new_id)
# parse the diff
parsed_diff = whatthepatch.parse_patch(diff_text)
# Our changed files
changed_files = set()
# Iterate over the diff -- one 'diff' per file
for diff in parsed_diff:
# Detect if the file is new or moved
if diff.header.old_path != diff.header.new_path:
if not self.allow_moves:
raise RuntimeError(
"Your commit range contains file moves. Cowardly aborting."
)
git_old_path = diff.header.old_path
# If the old path is not /dev/null, then ...
if git_old_path != "/dev/null":
# ... file moves start with 'a' (for old) and 'b' for new
old_leading_dir = git_old_path.split(os.path.sep, 1)[0]
assert old_leading_dir == "a"
git_new_path = diff.header.new_path
new_leading_dir, new_path = git_new_path.split(
os.path.sep, 1)
assert new_leading_dir == "b"
else:
# Otherwise, must be a wholly new file
new_path = diff.header.new_path
#
# TODO: we need to flag to the user that the Manage project
# needs updating and things _will not work_ because the units
# have moved
#
# Make it clear that this is further refined by interrogating Manage
else:
new_path = diff.header.new_path
# Currently, environments depend on the new files and the old files
changed_files.add(new_path)
return changed_files
def test_diff_unified_patchutil(self):
with open('tests/casefiles/diff-unified.diff') as f:
diff_text = f.read()
diff = next(wtp.parse_patch(diff_text))
new_text = wtp.apply.apply_diff(diff, self.lao, use_patch=True)
self.assertEqual(new_text, (self.tzu, None))
self.assertRaises(AssertionError, wtp.apply.apply_diff, diff, [''] + self.lao, use_patch=True)
def expectedRange(start_line, end_line, diff_str, file_path):
in_diff = False
for diff in whatthepatch.parse_patch(diff_str):
diff_path = diff[0].new_path
if diff_path == file_path:
in_diff = True
line_mapping = diff[1]
churn_cnt, new_start, new_end = mapLineNumber(
line_mapping, start_line, end_line)
return churn_cnt, new_start, new_end
return 0, start_line, end_line
def line_match(commit, traceback_line):
"""Return true if line_number was added to filename in commit"""
cmd = 'git', 'log', '-1', '--format=', '-p', str(commit)
diff = run_command(*cmd)
for diff in whatthepatch.parse_patch(diff):
if diff.header.new_path == traceback_line.git_filename:
for line in diff.changes:
if line[0] is None:
if line[1] == traceback_line.line_number:
return True
return False
def line_match(commit, traceback_line):
"""Return true if line_number was added to filename in commit"""
cmd = 'git', 'log', '-1', '--format=', '-p', str(commit)
diff = run_command(*cmd)
for diff in whatthepatch.parse_patch(diff):
if diff.header.new_path == traceback_line.git_filename:
for line in diff.changes:
if line[0] is None:
if line[1] == traceback_line.line_number:
return True
return False
def get_touched(self, patch):
res = {}
for diff in wtp.parse_patch(patch):
h = diff.header
new_p = h.new_path[2:] if h.new_path.startswith('b/') else h.new_p
res[new_p] = added = []
if diff.changes:
for old_line, new_line, _ in diff.changes:
if not old_line and new_line:
added.append(new_line)
return res
def __init__(self, patch, strip_dir=0, chdir=None):
self.root_dir = None
self.file_lookup = None
self.strip_dir = strip_dir
self.chdir = chdir
self.diffs = []
self.files = set()
for diff in whatthepatch.parse_patch(patch):
fn = self._diff_path(diff)
self.files.add(fn)
self.diffs.append(diff)
def createBugMap(projectName):
file = open('../projectData/filteredGitLogFor' + projectName + '.json',
'r')
filteredGitLog = json.load(file)
bugMap = {}
GitCommands().checkout('master')
count = 0
for k, v in list(filteredGitLog.items())[:]:
currentCommit, parentCommit = v[0]
patch = GitCommands().getDiff(parentCommit, currentCommit)
GitCommands().checkout(parentCommit)
# print(getGitLog())
# # print(checkoutMaster())
# print(getHeadHash())
for diff in whatthepatch.parse_patch(patch):
fileName = diff.header.old_path.split('/')[-1]
if fileName.split(".")[-1] == 'java':
if diff.changes == None:
continue
for prevLine, currLine, text in diff.changes:
if currLine == None:
gitBlameText = GitCommands().gitBlame(
diff.header.old_path, prevLine, prevLine)
if gitBlameText != None:
parsedData = GitCommands().parseGitBlame(
gitBlameText)
if parentCommit not in bugMap:
bugMap[parentCommit] = {}
if diff.header.old_path not in bugMap[
parentCommit]:
bugMap[parentCommit][diff.header.old_path] = [
(prevLine, parsedData[0], parsedData[1])
]
else:
bugMap[parentCommit][
diff.header.old_path].append(
(prevLine, parsedData[0],
parsedData[1]))
GitCommands().checkout('master')
count += 1
print("current count is at {}".format(count))
return bugMap
def test_diff_ed(self):
self.maxDiff = None
with open('tests/casefiles/diff-ed.diff') as f:
diff_text = f.read()
diff = next(wtp.parse_patch(diff_text))
new_text = wtp.apply.apply_diff(diff, self.lao)
self.assertEqual(self.tzu,new_text)
new_text = wtp.apply.apply_diff(diff, self.lao, use_patch=True)
self.assertEqual(new_text, (self.tzu, None))
def read_patch_data(patch_file_path):
try:
# noinspection PyPackageRequirements
import whatthepatch
except ImportError as e:
raise ImportError('The --use-patch feature requires the whatthepatch library. Run "pip install --force-reinstall mutmut[patch]"') from e
with open(patch_file_path) as f:
diffs = whatthepatch.parse_patch(f.read())
return {
diff.header.new_path: {change.new for change in diff.changes if change.old is None}
for diff in diffs
}
def test_svn_git_patch(self):
with open('tests/casefiles/svn-git.patch') as f:
text = f.read()
lines = text.splitlines()
csc_diff = diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/csc.py',
old_path='projects/bugs/bugtrace/trunk/src/bugtrace/csc.py',
old_version=12783,
new_path='projects/bugs/bugtrace/trunk/src/bugtrace/csc.py',
new_version=12784,
),
changes=CSC_CHANGES,
text='\n'.join(lines[:23]) + '\n'
)
diffxplore_path = 'bugtrace/trunk/src/bugtrace/Diffxplore.py'
diffxplore_diff = diffobj(
header=headerobj(
index_path=diffxplore_path,
old_path='projects/bugs/' + diffxplore_path,
old_version=12783,
new_path='projects/bugs/' + diffxplore_path,
new_version=12784,
),
changes=DIFFXPLORE_CHANGES,
text='\n'.join(lines[23:42]) + '\n'
)
bugexplore_path = 'bugtrace/trunk/src/bugtrace/Bugxplore.py'
bugxplore_diff = diffobj(
header=headerobj(
index_path=bugexplore_path,
old_path='projects/bugs/' + bugexplore_path,
old_version=12783,
new_path='projects/bugs/' + bugexplore_path,
new_version=12784,
),
changes=BUGXPLORE_CHANGES,
text='\n'.join(lines[42:]) + '\n'
)
expected = [csc_diff, diffxplore_diff, bugxplore_diff]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_svn_git_patch(self):
with open("tests/casefiles/svn-git.patch") as f:
text = f.read()
lines = text.splitlines()
csc_diff = diffobj(
header=headerobj(
index_path="bugtrace/trunk/src/bugtrace/csc.py",
old_path="projects/bugs/bugtrace/trunk/src/bugtrace/csc.py",
old_version=12783,
new_path="projects/bugs/bugtrace/trunk/src/bugtrace/csc.py",
new_version=12784,
),
changes=CSC_CHANGES,
text="\n".join(lines[:23]) + "\n",
)
diffxplore_path = "bugtrace/trunk/src/bugtrace/Diffxplore.py"
diffxplore_diff = diffobj(
header=headerobj(
index_path=diffxplore_path,
old_path="projects/bugs/" + diffxplore_path,
old_version=12783,
new_path="projects/bugs/" + diffxplore_path,
new_version=12784,
),
changes=DIFFXPLORE_CHANGES,
text="\n".join(lines[23:42]) + "\n",
)
bugexplore_path = "bugtrace/trunk/src/bugtrace/Bugxplore.py"
bugxplore_diff = diffobj(
header=headerobj(
index_path=bugexplore_path,
old_path="projects/bugs/" + bugexplore_path,
old_version=12783,
new_path="projects/bugs/" + bugexplore_path,
new_version=12784,
),
changes=BUGXPLORE_CHANGES,
text="\n".join(lines[42:]) + "\n",
)
expected = [csc_diff, diffxplore_diff, bugxplore_diff]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def commit_autosuggestions(diffs, endpoint):
commit_message = {}
for idx, example in enumerate(whatthepatch.parse_patch(diffs)):
if not example.changes:
continue
isadded, isdeleted = False, False
added, deleted = [], []
for change in example.changes:
if change.old == None and change.new != None:
added.append(change.line)
isadded = True
elif change.old != None and change.new == None:
deleted.append(change.line)
isdeleted = True
# To speed up tokenizing request.
added = tokenizing(" ".join(added), endpoint=endpoint)
deleted = tokenizing(" ".join(deleted), endpoint=endpoint)
_path = example.header.new_path \
if example.header.new_path \
else example.header.old_path
if _path:
if isadded and not isdeleted:
data = {"idx": idx, "added": added, "deleted": deleted}
res = requests.post(f'{endpoint}/added',
data=json.dumps(data),
headers={
'Content-Type':
'application/json; charset=utf-8'
})
commit = json.loads(res.text)
commit_message[_path] = commit
else:
data = {"idx": idx, "added": added, "deleted": deleted}
res = requests.post(f'{endpoint}/diff',
data=json.dumps(data),
headers={
'Content-Type':
'application/json; charset=utf-8'
})
commit = json.loads(res.text)
commit_message[_path] = commit
return commit_message
def test_svn_context_patch(self):
with open("tests/casefiles/svn-context.patch") as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(
header=headerobj(
index_path="bugtrace/trunk/src/bugtrace/csc.py",
old_path="bugtrace/trunk/src/bugtrace/csc.py",
old_version=12783,
new_path="bugtrace/trunk/src/bugtrace/csc.py",
new_version=12784,
),
changes=CSC_CHANGES,
text="\n".join(lines[:32]) + "\n",
),
diffobj(
header=headerobj(
index_path="bugtrace/trunk/src/bugtrace/Diffxplore.py",
old_path="bugtrace/trunk/src/bugtrace/Diffxplore.py",
old_version=12783,
new_path="bugtrace/trunk/src/bugtrace/Diffxplore.py",
new_version=12784,
),
changes=DIFFXPLORE_CHANGES,
text="\n".join(lines[32:61]) + "\n",
),
diffobj(
header=headerobj(
index_path="bugtrace/trunk/src/bugtrace/Bugxplore.py",
old_path="bugtrace/trunk/src/bugtrace/Bugxplore.py",
old_version=12783,
new_path="bugtrace/trunk/src/bugtrace/Bugxplore.py",
new_version=12784,
),
changes=BUGXPLORE_CHANGES,
text="\n".join(lines[61:]) + "\n",
),
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_svn_context_patch(self):
with open('tests/casefiles/svn-context.patch') as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/csc.py',
old_path='bugtrace/trunk/src/bugtrace/csc.py',
old_version=12783,
new_path='bugtrace/trunk/src/bugtrace/csc.py',
new_version=12784,
),
changes=CSC_CHANGES,
text='\n'.join(lines[:32]) + '\n'
),
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_version=12783,
new_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
new_version=12784,
),
changes=DIFFXPLORE_CHANGES,
text='\n'.join(lines[32:61]) + '\n'
),
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_version=12783,
new_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
new_version=12784,
),
changes=BUGXPLORE_CHANGES,
text='\n'.join(lines[61:]) + '\n'
),
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def run(patch):
# Initialize changes
changes = CommitDiff()
# Parse each file
for diff in whatthepatch.parse_patch(patch):
# Check file extenstion for .c or .h
filename = diff[0][3]
if not Parser.checkFileExtension(filename):
continue
# Create new file diff
file = Parser.parseFileChanges(diff)
# Add file changes to overall diff
changes.addFile(file)
return changes
def test_git_oneline_rm(self):
with open('tests/casefiles/git-oneline-rm.diff') as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(header=headerobj(index_path=None,
old_path='oneline.txt',
old_version='169ceeb',
new_path='/dev/null',
new_version='0000000'),
changes=[(1, None, 'Changed a one-line file.')],
text='\n'.join(lines[:34]) + '\n')
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def load_patch(patch_text):
patch = [diff for diff in whatthepatch.parse_patch(patch_text)]
status_header = collections.namedtuple(
'header', whatthepatch.patch.header._fields + ('path', 'status'))
patch = [
diff._replace(header=status_header(index_path=diff.header.index_path,
old_path=diff.header.old_path,
old_version=diff.header.old_version,
new_path=diff.header.new_path,
new_version=diff.header.new_version,
path=diff.header.new_path,
status='unchanged'))
for diff in patch
]
return patch
def get_potential_vccs(pygithub, apitoken, reponame, vfc_id):
# Get repo
repo = pygithub.get_repo(reponame)
# Get changed files in vulerability fixing commit
vfc = repo.get_commit(vfc_id)
vfc_raw = vfc.raw_data
vfc_files = vfc_raw['files']
vfc_oid = vfc_raw['sha']
# Get additions, deletions and replacements patch
candidates = []
for f in vfc_files:
# Parse patches and only keep added/removed/modified lines
wtp = [i for i in parse_patch(f['patch'])][0].changes
diffs = pd.DataFrame(wtp)
rmdiffs = diffs[(diffs.new.isna()) | (diffs.old.isna())]
# Skip if there are no removed or modified lines in file
if (len(rmdiffs.old.dropna()) < 1): continue
# Get blame of file on left side
blame = get_blame(reponame, vfc_oid, f['filename'], apitoken)
blamedf = pd.DataFrame(pd.json_normalize(blame))
# Get commit associated with remove/modified line
blame_commits = []
for rline in rmdiffs.old.dropna().tolist():
bcommit = blamedf[(blamedf.startingLine <= rline)
& (blamedf.endingLine >= rline)]
bcommit['old'] = rline
blame_commits.append(bcommit)
blamecommitsdf = pd.concat(blame_commits).set_index('old')
final = rmdiffs.set_index('old').join(blamecommitsdf).reset_index()
# Set meta info and add to candidate info
final['Filename'] = f['filename']
final['Status'] = f['status']
candidates.append(final)
return pd.concat(candidates)
def line_removed(target_line, commit):
"""Given a commit tell if target_line was added or removed.
True if line was removed
False if added
None if target_line wasn't found at all (because not a full line etc.)
"""
cmd = 'git', 'log', '-1', '--format=', '-p', str(commit)
diff = run_command(*cmd)
for diff in whatthepatch.parse_patch(diff):
for line in diff.changes:
if target_line in line[2]:
if line[0] is None:
# Line added
return False
elif line[1] is None:
# Line removed
return True
# target_line matched part of a line instead of a full line
return None
def line_removed(target_line, commit):
"""Given a commit tell if target_line was added or removed.
True if line was removed
False if added
None if target_line wasn't found at all (because not a full line etc.)
"""
cmd = 'git', 'log', '-1', '--format=', '-p', str(commit)
diff = run_command(*cmd)
for diff in whatthepatch.parse_patch(diff):
for line in diff.changes:
if target_line in line[2]:
if line[0] is None:
# Line added
return False
elif line[1] is None:
# Line removed
return True
# target_line matched part of a line instead of a full line
return None
def test_git_new_empty_file(self):
with open('tests/casefiles/git-new-empty-file.diff') as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(header=headerobj(
index_path=None,
old_path='/dev/null',
old_version='0000000',
new_path='somefile.txt',
new_version='e69de29',
),
changes=[],
text='\n'.join(lines[:34]) + '\n')
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def __change_color(self, orig_src, new_src, diff):
# Step 1. Parsing patch
diff_list = whatthepatch.parse_patch(diff)
# Step 2. Iterate each of patch file (it may be more than one)
for diff in diff_list:
for s, d, v in diff.changes:
# Step 1. Orig have no line (Insert new line)
if s is None:
new_src.set_line_class(d, "bg-success")
# Step 2. New habe no line (Remove exist line)
elif d is None:
orig_src.set_line_class(s, "bg-danger")
# Step 3. Other change
elif s != d:
orig_src.set_line_class(s, "bg-info")
new_src.set_line_class(d, "bg-info")
elif s == d:
pass
else:
pass
def _get_modified_lines(_file):
# Gets the diff from GIT
diff_patch = subprocess.check_output(
'git diff --cached %s' % _file,
shell=True
)
# Parse the patch-format and get only added or modified
diff = [x for x in whatthepatch.parse_patch(diff_patch)]
if not diff:
return []
errors = []
for old, new, content in diff[0].changes:
if old is None:
errors.append(new)
if content == '': # if content is blank line, add next line
errors.append(new + 1)
return errors
def read_patch_data(patch_file_path):
print('Using patch data from ' + patch_file_path)
try:
# noinspection PyPackageRequirements
import whatthepatch
except ImportError:
print(
'The --use-patch feature requires the whatthepatch library. Run "pip install whatthepatch"',
file=sys.stderr)
raise
with open(patch_file_path) as f:
diffs = whatthepatch.parse_patch(f.read())
return {
diff.header.new_path: {
line_number
for old_line_number, line_number, text in diff.changes
if old_line_number is None
}
for diff in diffs
}
def load_patch(patch_text):
patch = [diff for diff in whatthepatch.parse_patch(patch_text)]
status_header = collections.namedtuple(
'header',
whatthepatch.patch.header._fields + ('path', 'status')
)
patch = [
diff._replace(header=status_header(
index_path=diff.header.index_path,
old_path=diff.header.old_path,
old_version=diff.header.old_version,
new_path=diff.header.new_path,
new_version=diff.header.new_version,
path=diff.header.new_path,
status='unchanged'
))
for diff in patch]
return patch
def analyze_diff(self, diff):
def status(change):
if change[0] is None:
return 'insert'
elif change[1] is None:
return 'delete'
else:
return 'equal'
out = []
for hunk in whatthepatch.parse_patch(diff.decode('utf-8')):
file_changes = []
in_changeset = False
for mode in [('insert',1), ('delete',0)]:
for change in hunk.changes:
if status(change) == mode[0] and not in_changeset:
in_changeset = True
file_changes.append({'start':change[mode[1]], 'type':mode[0]})
elif status(change) == 'equal' and in_changeset:
in_changeset = False
file_changes[-1]['end'] = change[mode[1]]-1
out.append({'header':hunk.header, 'changes':file_changes})
return out
def test_svn_unified_patch(self):
with open('tests/casefiles/svn-unified.patch') as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/csc.py',
old_path='bugtrace/trunk/src/bugtrace/csc.py',
old_version=12783,
new_path='bugtrace/trunk/src/bugtrace/csc.py',
new_version=12784,
),
changes=CSC_CHANGES,
text='\n'.join(lines[:22]) + '\n'),
diffobj(header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_version=12783,
new_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
new_version=12784,
),
changes=DIFFXPLORE_CHANGES,
text='\n'.join(lines[22:40]) + '\n'),
diffobj(header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_version=12783,
new_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
new_version=12784,
),
changes=BUGXPLORE_CHANGES,
text='\n'.join(lines[40:]) + '\n')
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_git_new_empty_file(self):
with open('tests/casefiles/git-new-empty-file.diff') as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(
header=headerobj(
index_path=None,
old_path='/dev/null',
old_version='0000000',
new_path='somefile.txt',
new_version='e69de29',
),
changes=[],
text='\n'.join(lines[:34]) + '\n'
)
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_git_new_empty_file(self):
with open("tests/casefiles/git-new-empty-file.diff") as f:
text = f.read()
lines = text.splitlines()
expected = [
diffobj(
header=headerobj(
index_path=None,
old_path="/dev/null",
old_version="0000000",
new_path="somefile.txt",
new_version="e69de29",
),
changes=[],
text="\n".join(lines[:34]) + "\n",
)
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def patch_analysis(self, patch):
info = {'size': 0, 'test_size': 0, 'addlines': 0, 'rmlines': 0}
for diff in whatthepatch.parse_patch(patch):
if diff.header and diff.changes:
h = diff.header
new_path = h.new_path[2:] if h.new_path.startswith(
'b/') else h.new_path
# Calc changes additions & deletions
counts = [(old is None and new is not None, new is None
and old is not None)
for old, new, _ in diff.changes]
counts = list(zip(*counts)) # inverse zip
info['addlines'] += sum(counts[0])
info['rmlines'] += sum(counts[1])
if utils.is_test_file(new_path):
info['test_size'] += len(diff.changes)
else:
info['size'] += len(diff.changes)
return info
def patch_analysis(self, patch):
info = {'size': 0, 'test_size': 0, 'addlines': 0, 'rmlines': 0}
for diff in whatthepatch.parse_patch(patch):
if diff.header and diff.changes:
h = diff.header
new_path = h.new_path[2:] if h.new_path.startswith('b/') else h.new_path
# Calc changes additions & deletions
counts = [
(old is None and new is not None, new is None and old is not None)
for old, new, _ in diff.changes
]
counts = list(zip(*counts)) # inverse zip
info['addlines'] += sum(counts[0])
info['rmlines'] += sum(counts[1])
if utils.is_test_file(new_path):
info['test_size'] += len(diff.changes)
else:
info['size'] += len(diff.changes)
return info
def get_commits_from_patch(patch, args):
"""Get the set of commits associated with added lines in the patch."""
commits = set()
for diff in whatthepatch.parse_patch(patch):
for change in diff.changes:
# Ignore lines that aren't insertions
if change.old is not None or change.new is None:
continue
cmd = ('git', '-C', args.repo, 'blame',
'-L{0},{0}'.format(change.new), args.modified_rev,
diff.header.new_path)
output = subprocess.run(cmd,
capture_output=True,
check=True,
text=True)
# Output looks like this:
#
# f9aa76a852485 (Dave Airlie 2012-04-17 14:12:29 +0100 132) .name = DRIVER_NAME,
#
# Discard everything from the first close paren on, then
# get the commit hash and the data portion of the
# output. (I'm not sure whether this is author date or
# commit date.)
header = output.stdout.split(')')[0]
parts = header.split()
commit_hash = parts[0]
commit_date = parts[-4]
commit_time = parts[-3]
commit_zone = parts[-2]
commit_datetime = '{}T{}{}'.format(commit_date, commit_time,
commit_zone)
commits.add((commit_datetime, commit_hash))
return commits
def _apply(src, diff_text, reverse=False, use_patch=False):
diff = next(wtp.parse_patch(diff_text))
return wtp.apply.apply_diff(diff, src, reverse, use_patch)
def test_svn_rcs_patch(self):
with open('tests/casefiles/svn-rcs.patch') as f:
text = f.read()
lines = text.splitlines()
csc_changes = [
(None, 1, '# This is a basic script I wrote to run '
'Bugxplore over the dataset'),
(None, 2, ''),
(None, 3, ''),
(8, None, None),
(9, None, None),
(None, 11, 'from Bugxplore import main'),
(None, 12, 'from Bugxplore import _make_dir'),
]
diffxplore_changes = [
(49, None, None),
(None, 49, " optparser.set_defaults(output_dir='/tmp/diffs')"),
(53, None, None),
(None, 53, ''),
]
bugxplore_changes = [
(86, None, None),
(None, 86, " optparser.set_defaults(output_dir='/tmp/bugs')"),
(91, None, None),
(None, 91, ''),
]
expected = [
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/csc.py',
old_path='bugtrace/trunk/src/bugtrace/csc.py',
old_version=None,
new_path='bugtrace/trunk/src/bugtrace/csc.py',
new_version=None,
),
changes=csc_changes,
text='\n'.join(lines[:10]) + '\n'
),
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_version=None,
new_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
new_version=None,
),
changes=diffxplore_changes,
text='\n'.join(lines[10:18]) + '\n'
),
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_version=None,
new_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
new_version=None,
),
changes=bugxplore_changes,
text='\n'.join(lines[18:]) + '\n'
),
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_svn_default_patch(self):
with open('tests/casefiles/svn-default.patch') as f:
text = f.read()
lines = text.splitlines()
csc_changes = [
(None, 1, '# This is a basic script I wrote to run '
'Bugxplore over the dataset'),
(None, 2, ''),
(None, 3, ''),
(8, None, 'from Main import main'),
(9, None, 'from Main import _make_dir'),
(None, 11, 'from Bugxplore import main'),
(None, 12, 'from Bugxplore import _make_dir'),
]
diffxplore_changes = indent(4, [
(49, None, "optparser.set_defaults(output_dir='/tmp/sctdiffs',"
"project_name='default_project')"),
(None, 49, "optparser.set_defaults(output_dir='/tmp/diffs')"),
(53, None, "optparser.add_option('-a', '--append', "
"action='store_true', dest='app', default=False, "
"help='Append to existing MethTerms2 document')"),
(None, 53, ''),
])
bugxplore_changes = indent(4, [
(86, None, "optparser.set_defaults(output_dir='/tmp/bugs',"
"project_name='default_project')"),
(None, 86, "optparser.set_defaults(output_dir='/tmp/bugs')"),
(91, None, "optparser.add_option('-a', '--append', "
"action='store_true', dest='app', default=False, "
"help='Append to existing MethTerms2 document')"),
(None, 91, ''),
])
expected = [
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/csc.py',
old_path='bugtrace/trunk/src/bugtrace/csc.py',
old_version=None,
new_path='bugtrace/trunk/src/bugtrace/csc.py',
new_version=None,
),
changes=csc_changes,
text='\n'.join(lines[:12]) + '\n'
),
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
old_version=None,
new_path='bugtrace/trunk/src/bugtrace/Diffxplore.py',
new_version=None,
),
changes=diffxplore_changes,
text='\n'.join(lines[12:22]) + '\n'
),
diffobj(
header=headerobj(
index_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
old_version=None,
new_path='bugtrace/trunk/src/bugtrace/Bugxplore.py',
new_version=None,
),
changes=bugxplore_changes,
text='\n'.join(lines[22:]) + '\n'
)
]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def test_git_patch(self):
with open('tests/casefiles/git.patch') as f:
text = f.read()
lines = text.splitlines()
novel_frame_changes = indent(4, [
(135, 135, 'public void actionPerformed(ActionEvent e) {'),
(136, 136, ''),
(137, 137, ' if (e.getActionCommand().equals("OPEN")) {'),
(138, None, ' prefsDialog(prefs.getImportPane());'),
(None, 138, ' prefs.selectImportPane();'),
(None, 139, ' prefsDialog();'),
(139, 140, ' } else if (e.getActionCommand().equals("SET")) {'),
(140, None, ' prefsDialog(prefs.getRepoPane());'),
(None, 141, ' prefs.selectRepoPane();'),
(None, 142, ' prefsDialog();'),
(141, 143, ' } else if (e.getActionCommand().equals("PREFS"))'),
(142, 144, ' prefsDialog();'),
(143, 145, ' else if (e.getActionCommand().equals("EXIT"))'),
(158, 160, ' * Create dialog to handle user preferences'),
(159, 161, ' */'),
(160, 162, 'public void prefsDialog() {'),
(161, None, ''),
(162, 163, ' prefs.setVisible(true);'),
(163, 164, '}'),
(164, 165, ''),
(165, None, 'public void prefsDialog(Component c) {'),
(166, None, ' prefs.setSelectedComponent(c);'),
(167, None, ' prefsDialog();'),
(168, None, '}'),
(169, None, ''),
(170, 166, '/**'),
(171, 167, ' * Open software tutorials, '
'most likely to be hosted online'),
(172, 168, ' * ')
])
novel_frame_path = (
'novel/src/java/edu/ua/eng/software/novel/NovelFrame.java'
)
novel_frame = diffobj(
header=headerobj(
index_path=None,
old_path=novel_frame_path,
old_version='aae63fe',
new_path=novel_frame_path,
new_version='5abbc99'
),
changes=novel_frame_changes,
text='\n'.join(lines[:34]) + '\n'
)
novel_pref_frame_path = (
'novel/src/java/edu/ua/eng/software/novel/NovelPrefPane.java'
)
novel_pref_frame = diffobj(
header=headerobj(
index_path=None,
old_path=novel_pref_frame_path,
old_version='a63b57e',
new_path=novel_pref_frame_path,
new_version='919f413'
),
changes=[
(18, 18, ''),
(19, 19, ' public abstract void apply();'),
(20, 20, ''),
(None, 21, ' public abstract void applyPrefs();'),
(None, 22, ''),
(21, 23, ' public abstract boolean isChanged();'),
(22, 24, ''),
(23, 25, ' protected Preferences prefs;')],
text='\n'.join(lines[34:]) + '\n',
)
expected = [novel_frame, novel_pref_frame]
results = list(wtp.parse_patch(text))
self.assert_diffs_equal(results, expected)
def post(self, request):
return self.render_to_response({'diffs': whatthepatch.parse_patch(request.POST['diff'])})