def test_no_newline_both(self):
patch = """diff --git a/test b/test
index d800886..bed2d6a 100644
--- a/test
+++ b/test
@@ -1 +1 @@
-123
\ No newline at end of file
+123n
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert (
diff
== """
--- a/test
+++ b/test
@@ -1 +1 @@
-123
\ No newline at end of file
+123n
\ No newline at end of file
"""
)
def get(self, build_id):
build = Build.query.get(build_id)
if build is None:
return '', 404
args = self.parser.parse_args()
results = get_coverage_by_build_id(build.id)
if args.diff:
diff = build.source.generate_diff()
if not diff:
return self.respond({})
diff_parser = DiffParser(diff)
parsed_diff = diff_parser.parse()
files_in_diff = set(
d['new_filename'][2:] for d in parsed_diff
if d['new_filename']
)
results = [r for r in results if r.filename in files_in_diff]
coverage = {
c.filename: {
'linesCovered': c.lines_covered,
'linesUncovered': c.lines_uncovered,
'diffLinesCovered': c.diff_lines_covered,
'diffLinesUncovered': c.diff_lines_uncovered,
}
for c in results
}
return self.respond(coverage)
def get(self, build_id):
build = Build.query.get(build_id)
if build is None:
return '', 404
args = self.parser.parse_args()
results = get_coverage_by_build_id(build.id)
if args.diff:
diff = build.source.generate_diff()
if not diff:
return self.respond({})
diff_parser = DiffParser(diff)
lines_by_file = diff_parser.get_lines_by_file()
results = [r for r in results if r.filename in lines_by_file]
coverage_data = merged_coverage_data(results)
coverage_stats = {}
for filename in lines_by_file:
if filename in coverage_data and filename in lines_by_file:
stats = get_coverage_stats(lines_by_file[filename], coverage_data[filename])
coverage_stats[filename] = {
'linesCovered': stats.lines_covered,
'linesUncovered': stats.lines_uncovered,
'diffLinesCovered': stats.diff_lines_covered,
'diffLinesUncovered': stats.diff_lines_uncovered,
}
else:
# NOTE: Without a diff, the stats may be off if there are
# multiple job steps. (Each job step can potentially
# return a separate FileCoverage row for the same file.)
# For each file, we return the best metrics using
# min()/max(); if you want more correct metrics, pass
# diff=1.
coverage_stats = {}
for r in results:
if r.filename not in coverage_stats:
coverage_stats[r.filename] = {
'linesCovered': r.lines_covered,
'linesUncovered': r.lines_uncovered,
'diffLinesCovered': r.diff_lines_covered,
'diffLinesUncovered': r.diff_lines_uncovered,
}
else:
# Combine metrics using max() for [diff] lines
# covered, min() for [diff] lines uncovered.
stats = coverage_stats[r.filename]
coverage_stats[r.filename] = {
'linesCovered': max(stats['linesCovered'], r.lines_covered),
'linesUncovered': min(stats['linesUncovered'], r.lines_uncovered),
'diffLinesCovered': max(stats['diffLinesCovered'], r.diff_lines_covered),
'diffLinesUncovered': min(stats['diffLinesUncovered'], r.diff_lines_uncovered),
}
return self.respond(coverage_stats)
def test_get_changed_files_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.get_changed_files()
assert files == set([
'changes/utils/diff_parser.py',
])
lines_by_file = parser.get_lines_by_file()
assert lines_by_file == {'changes/utils/diff_parser.py': {74}}
def test_get_changed_files_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.get_changed_files()
assert files == set([
'changes/utils/diff_parser.py',
])
lines_by_file = parser.get_lines_by_file()
assert lines_by_file == {'changes/utils/diff_parser.py': {74}}
def get_changed_files(self):
vcs = self.repository.get_vcs()
if not vcs:
raise NotImplementedError
diff = vcs.export(self.revision.sha)
diff_parser = DiffParser(diff)
return diff_parser.get_changed_files()
def test_get_changed_files_complex_diff(self):
parser = DiffParser(COMPLEX_DIFF)
files = parser.get_changed_files()
assert files == set(["ci/run_with_retries.py", "ci/server-collect", "ci/not-real"])
lines_by_file = parser.get_lines_by_file()
assert set(lines_by_file) == files
assert lines_by_file["ci/not-real"] == {1}
assert lines_by_file["ci/server-collect"] == {24, 31, 39, 46}
assert lines_by_file["ci/run_with_retries.py"] == {2, 45} | set(range(53, 63)) | set(range(185, 192))
def test_parse_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert files == [
{
"old_filename": "a/changes/utils/diff_parser.py",
"new_filename": "b/changes/utils/diff_parser.py",
"chunk_markers": ["@@ -71,6 +71,7 @@ class DiffParser(object):"],
"chunks": [
[
{
"action": "unmod",
"line": " continue",
"new_lineno": 71,
"old_lineno": 71,
"ends_with_newline": True,
},
{"action": "unmod", "line": "", "new_lineno": 72, "old_lineno": 72, "ends_with_newline": True},
{
"action": "unmod",
"line": " chunks = []",
"new_lineno": 73,
"old_lineno": 73,
"ends_with_newline": True,
},
{
"action": "add",
"line": " chunk_markers = []",
"new_lineno": 74,
"old_lineno": u"",
"ends_with_newline": True,
},
{
"action": "unmod",
"line": " old, new = self._extract_rev(line, lineiter.next())",
"new_lineno": 75,
"old_lineno": 74,
"ends_with_newline": True,
},
{
"action": "unmod",
"line": " files.append({",
"new_lineno": 76,
"old_lineno": 75,
"ends_with_newline": True,
},
{
"action": "unmod",
"line": " 'old_filename': old[0] if old[0] != '/dev/null' else None,",
"new_lineno": 77,
"old_lineno": 76,
"ends_with_newline": True,
},
]
],
}
]
def test_parse_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert files == [
FileInfo(
old_filename='a/changes/utils/diff_parser.py',
new_filename='b/changes/utils/diff_parser.py',
chunk_markers=['@@ -71,6 +71,7 @@ class DiffParser(object):'],
chunks=[[
LineInfo(
action='unmod',
line=' continue',
new_lineno=71,
old_lineno=71,
ends_with_newline=True),
LineInfo(
action='unmod',
line='',
new_lineno=72,
old_lineno=72,
ends_with_newline=True),
LineInfo(
action='unmod',
line=' chunks = []',
new_lineno=73,
old_lineno=73,
ends_with_newline=True),
LineInfo(
action='add',
line=' chunk_markers = []',
new_lineno=74,
old_lineno=0,
ends_with_newline=True),
LineInfo(
action='unmod',
line=' old, new = self._extract_rev(line, lineiter.next())',
new_lineno=75,
old_lineno=74,
ends_with_newline=True),
LineInfo(
action='unmod',
line=' files.append({',
new_lineno=76,
old_lineno=75,
ends_with_newline=True),
LineInfo(
action='unmod',
line=" 'old_filename': old[0] if old[0] != '/dev/null' else None,",
new_lineno=77,
old_lineno=76,
ends_with_newline=True)
]],
)
]
def test_add_empty_file(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
new file mode 100644
index 0000000..e69de29
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == ""
assert file_dict["old_filename"] is None
assert parser.get_changed_files() == set(["diff-from/__init__.py"])
assert parser.get_lines_by_file() == {}
def test_remove_empty_file(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
deleted file mode 100644
index e69de29..0000000
"""
parser = DiffParser(patch)
(file_info,) = parser.parse()
diff = parser.reconstruct_file_diff(file_info)
assert diff == ""
assert file_info.new_filename is None
assert parser.get_changed_files() == set(['diff-from/__init__.py'])
assert parser.get_lines_by_file() == {}
def get_changed_files(self, id):
"""Returns the list of files changed in a revision.
Args:
id (str): The id of the revision.
Returns:
A set of filenames
Raises:
UnknownRevision: If the revision wan't found.
"""
diff = self.export(id)
diff_parser = DiffParser(diff)
return diff_parser.get_changed_files()
def get_changed_files(self, id):
"""Returns the list of files changed in a revision.
Args:
id (str): The id of the revision.
Returns:
A set of filenames
Raises:
UnknownRevision: If the revision wan't found.
"""
diff = self.export(id)
diff_parser = DiffParser(diff)
return diff_parser.get_changed_files()
def test_no_newline_empty_source(self):
patch = """diff --git a/test b/test
index e69de29..d800886 100644
--- a/test
+++ b/test
@@ -0,0 +1 @@
+123
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def test_no_newline_empty_target(self):
patch = """diff --git a/test b/test
index d800886..e69de29 100644
--- a/test
+++ b/test
@@ -1 +0,0 @@
-123
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def test_no_newline_empty_source(self):
patch = """diff --git a/test b/test
index e69de29..d800886 100644
--- a/test
+++ b/test
@@ -0,0 +1 @@
+123
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def test_no_newline_empty_target(self):
patch = """diff --git a/test b/test
index d800886..e69de29 100644
--- a/test
+++ b/test
@@ -1 +0,0 @@
-123
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def test_parse_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert files == [
FileInfo(
old_filename='a/changes/utils/diff_parser.py',
new_filename='b/changes/utils/diff_parser.py',
chunk_markers=['@@ -71,6 +71,7 @@ class DiffParser(object):'],
chunks=[[
LineInfo(action='unmod',
line=' continue',
new_lineno=71,
old_lineno=71,
ends_with_newline=True),
LineInfo(action='unmod',
line='',
new_lineno=72,
old_lineno=72,
ends_with_newline=True),
LineInfo(action='unmod',
line=' chunks = []',
new_lineno=73,
old_lineno=73,
ends_with_newline=True),
LineInfo(action='add',
line=' chunk_markers = []',
new_lineno=74,
old_lineno=0,
ends_with_newline=True),
LineInfo(
action='unmod',
line=
' old, new = self._extract_rev(line, lineiter.next())',
new_lineno=75,
old_lineno=74,
ends_with_newline=True),
LineInfo(action='unmod',
line=' files.append({',
new_lineno=76,
old_lineno=75,
ends_with_newline=True),
LineInfo(
action='unmod',
line=
" 'old_filename': old[0] if old[0] != '/dev/null' else None,",
new_lineno=77,
old_lineno=76,
ends_with_newline=True)
]],
)
]
def test_no_newline_target(self):
patch = """diff --git a/test b/test
index 190a180..d800886 100644
--- a/test
+++ b/test
@@ -1 +1 @@
-123
+123
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def process_diff(self):
lines_by_file = defaultdict(set)
try:
source = self.step.job.build.source
except AttributeError:
return lines_by_file
diff = source.generate_diff()
if not diff:
return lines_by_file
diff_parser = DiffParser(diff)
return diff_parser.get_lines_by_file()
def test_get_changed_files_complex_diff(self):
parser = DiffParser(COMPLEX_DIFF)
files = parser.get_changed_files()
assert files == set([
'ci/run_with_retries.py',
'ci/server-collect',
'ci/not-real',
])
lines_by_file = parser.get_lines_by_file()
assert set(lines_by_file) == files
assert lines_by_file['ci/not-real'] == {1}
assert lines_by_file['ci/server-collect'] == {24, 31, 39, 46}
assert lines_by_file['ci/run_with_retries.py'] == {2, 45} | set(
range(53, 63)) | set(range(185, 192))
def test_no_newline_source(self):
patch = """diff --git a/test b/test
index d800886..190a180 100644
--- a/test
+++ b/test
@@ -1 +1 @@
-123
\ No newline at end of file
+123
"""
parser = DiffParser(patch)
(file_info,) = parser.parse()
diff = parser.reconstruct_file_diff(file_info)
assert diff == """
def process_diff(self):
lines_by_file = defaultdict(set)
try:
source = self.step.job.build.source
except AttributeError:
return lines_by_file
diff = source.generate_diff()
if not diff:
return lines_by_file
diff_parser = DiffParser(diff)
return diff_parser.get_lines_by_file()
def test_no_newline_target(self):
patch = """diff --git a/test b/test
index 190a180..d800886 100644
--- a/test
+++ b/test
@@ -1 +1 @@
-123
+123
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def test_no_newline_source(self):
patch = """diff --git a/test b/test
index d800886..190a180 100644
--- a/test
+++ b/test
@@ -1 +1 @@
-123
\ No newline at end of file
+123
"""
parser = DiffParser(patch)
(file_info, ) = parser.parse()
diff = parser.reconstruct_file_diff(file_info)
assert diff == """
def test_no_newline_both(self):
patch = """diff --git a/test b/test
index d800886..bed2d6a 100644
--- a/test
+++ b/test
@@ -1 +1 @@
-123
\ No newline at end of file
+123n
\ No newline at end of file
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
def test_add_multiple_empty_files(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/diff-from/other.py b/diff-from/other.py
new file mode 100644
index 0000000..e69de29
"""
parser = DiffParser(patch)
(first_dict, second_dict) = parser.parse()
assert first_dict["new_filename"] == "b/diff-from/__init__.py"
assert first_dict["old_filename"] is None
assert second_dict["new_filename"] == "b/diff-from/other.py"
assert second_dict["old_filename"] is None
assert parser.get_changed_files() == set(["diff-from/__init__.py", "diff-from/other.py"])
assert parser.get_lines_by_file() == {}
def test_add_multiple_empty_files(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/diff-from/other.py b/diff-from/other.py
new file mode 100644
index 0000000..e69de29
"""
parser = DiffParser(patch)
(first_info, second_info,) = parser.parse()
assert first_info.new_filename == 'b/diff-from/__init__.py'
assert first_info.old_filename is None
assert second_info.new_filename == 'b/diff-from/other.py'
assert second_info.old_filename is None
assert parser.get_changed_files() == set(['diff-from/__init__.py', 'diff-from/other.py'])
assert parser.get_lines_by_file() == {}
def _get_revision_changed_files(repository, revision):
vcs = repository.get_vcs()
if not vcs:
raise NotImplementedError
try:
diff = vcs.export(revision.sha)
except UnknownRevision:
vcs.update()
try:
diff = vcs.export(revision.sha)
except UnknownRevision:
raise MissingRevision('Unable to find revision %s' % (revision.sha,))
diff_parser = DiffParser(diff)
return diff_parser.get_changed_files()
def get_changed_files(self):
vcs = self.repository.get_vcs()
if not vcs:
raise NotImplementedError
diff = vcs.export(self.revision.sha)
diff_parser = DiffParser(diff)
parsed_diff = diff_parser.parse()
results = set()
for info in parsed_diff:
if info['new_filename']:
results.add(info['new_filename'][2:])
if info['old_filename']:
results.add(info['old_filename'][2:])
return results
def _get_revision_changed_files(repository, revision):
vcs = repository.get_vcs()
if not vcs:
raise NotImplementedError
try:
diff = vcs.export(revision.sha)
except UnknownRevision:
vcs.update()
try:
diff = vcs.export(revision.sha)
except UnknownRevision:
raise MissingRevision('Unable to find revision %s' %
(revision.sha, ))
diff_parser = DiffParser(diff)
return diff_parser.get_changed_files()
def post(self, diff_id):
"""
Ask Changes to restart all builds for this diff. The response will be
the list of all builds.
"""
diff = self._get_diff_by_id(diff_id)
if not diff:
return error("Diff with ID %s does not exist." % (diff_id, ))
diff_parser = DiffParser(diff.source.patch.diff)
files_changed = diff_parser.get_changed_files()
try:
projects = self._get_projects_for_diff(diff, files_changed)
except InvalidDiffError:
return error('Patch does not apply')
except ProjectConfigError:
return error('Project config is not in a valid format.')
collection_id = uuid.uuid4()
builds = self._get_builds_for_diff(diff)
new_builds = []
for project in projects:
builds_for_project = [
x for x in builds if x.project_id == project.id
]
if not builds_for_project:
logging.warning('Project with id %s does not have a build.',
project.id)
continue
build = max(builds_for_project, key=lambda x: x.number)
if build.status is not Status.finished:
continue
if build.result is Result.passed:
continue
new_build = create_build(
project=project,
collection_id=collection_id,
label=build.label,
target=build.target,
message=build.message,
author=build.author,
source=diff.source,
cause=Cause.retry,
selective_testing_policy=build.selective_testing_policy,
)
new_builds.append(new_build)
return self.respond(new_builds)
def test_reconstruct_file_diff_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert len(files) == 1
diff = parser.reconstruct_file_diff(files[0])
correct = """
--- a/changes/utils/diff_parser.py
+++ b/changes/utils/diff_parser.py
@@ -71,6 +71,7 @@ class DiffParser(object):
""" + ' ' + """
in_header = False
chunks = []
+ chunk_markers = []
old, new = self._extract_rev(line, lineiter.next())
files.append({
'is_header': False,
"""
assert diff == correct
def test_reconstruct_file_diff_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert len(files) == 1
diff = parser.reconstruct_file_diff(files[0])
correct = """
--- a/changes/utils/diff_parser.py
+++ b/changes/utils/diff_parser.py
@@ -71,6 +71,7 @@ class DiffParser(object):
continue
""" + ' ' + """
chunks = []
+ chunk_markers = []
old, new = self._extract_rev(line, lineiter.next())
files.append({
'old_filename': old[0] if old[0] != '/dev/null' else None,
"""
assert diff == correct
def test_reconstruct_file_diff_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert len(files) == 1
diff = parser.reconstruct_file_diff(files[0])
correct = """
--- a/changes/utils/diff_parser.py
+++ b/changes/utils/diff_parser.py
@@ -71,6 +71,7 @@ class DiffParser(object):
continue
""" + ' ' + """
chunks = []
+ chunk_markers = []
old, new = self._extract_rev(line, lineiter.next())
files.append({
'old_filename': old[0] if old[0] != '/dev/null' else None,
"""
assert diff == correct
def test_reconstruct_file_diff_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert len(files) == 1
diff = parser.reconstruct_file_diff(files[0])
correct = """
--- a/changes/utils/diff_parser.py
+++ b/changes/utils/diff_parser.py
@@ -71,6 +71,7 @@ class DiffParser(object):
""" + ' ' + """
in_header = False
chunks = []
+ chunk_markers = []
old, new = self._extract_rev(line, lineiter.next())
files.append({
'is_header': False,
"""
assert diff == correct
def post(self, diff_id):
"""
Ask Changes to restart all builds for this diff. The response will be
the list of all builds.
"""
diff = self._get_diff_by_id(diff_id)
if not diff:
return error("Diff with ID %s does not exist." % (diff_id,))
diff_parser = DiffParser(diff.source.patch.diff)
files_changed = diff_parser.get_changed_files()
try:
projects = self._get_projects_for_diff(diff, files_changed)
except InvalidDiffError:
return error('Patch does not apply')
except ProjectConfigError:
return error('Project config is not in a valid format.')
collection_id = uuid.uuid4()
builds = self._get_builds_for_diff(diff)
new_builds = []
for project in projects:
builds_for_project = [x for x in builds if x.project_id == project.id]
if not builds_for_project:
logging.warning('Project with id %s does not have a build.', project.id)
continue
build = max(builds_for_project, key=lambda x: x.number)
if build.status is not Status.finished:
continue
if build.result is Result.passed:
continue
new_build = create_build(
project=project,
collection_id=collection_id,
label=build.label,
target=build.target,
message=build.message,
author=build.author,
source=diff.source,
cause=Cause.retry,
selective_testing_policy=build.selective_testing_policy,
)
new_builds.append(new_build)
return self.respond(new_builds)
def get_changed_files(self):
vcs = self.repository.get_vcs()
if not vcs:
raise NotImplementedError
# Make sure the repo exists on disk.
if not vcs.exists():
vcs.clone()
diff = None
try:
diff = vcs.export(self.revision.sha)
except UnknownRevision:
# Maybe the repo is stale; update.
vcs.update()
# If it doesn't work this time, we have
# a problem. Let the exception escape.
diff = vcs.export(self.revision.sha)
diff_parser = DiffParser(diff)
return diff_parser.get_changed_files()
def test_dev_null_target(self):
patch = """diff --git a/whitelist/blacklist/b.txt b/whitelist/blacklist/b.txt
deleted file mode 100644
index 038d718..0000000
--- a/whitelist/blacklist/b.txt
+++ /dev/null
@@ -1 +0,0 @@
-testing
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
--- a/whitelist/blacklist/b.txt
+++ /dev/null
@@ -1 +0,0 @@
-testing
"""
assert file_dict['new_filename'] is None
assert parser.get_changed_files() == set(['whitelist/blacklist/b.txt'])
def test_dev_null_source(self):
patch = """diff --git a/whitelist/blacklist/a.txt b/whitelist/blacklist/a.txt
new file mode 100644
index 0000000..038d718
--- /dev/null
+++ b/whitelist/blacklist/a.txt
@@ -0,0 +1 @@
+testing
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
--- /dev/null
+++ b/whitelist/blacklist/a.txt
@@ -0,0 +1 @@
+testing
"""
assert file_dict['old_filename'] is None
assert parser.get_changed_files() == set(['whitelist/blacklist/a.txt'])
def test_add_multiple_empty_files(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/diff-from/other.py b/diff-from/other.py
new file mode 100644
index 0000000..e69de29
"""
parser = DiffParser(patch)
(
first_info,
second_info,
) = parser.parse()
assert first_info.new_filename == 'b/diff-from/__init__.py'
assert first_info.old_filename is None
assert second_info.new_filename == 'b/diff-from/other.py'
assert second_info.old_filename is None
assert parser.get_changed_files() == set(
['diff-from/__init__.py', 'diff-from/other.py'])
assert parser.get_lines_by_file() == {}
def test_dev_null_target(self):
patch = """diff --git a/whitelist/blacklist/b.txt b/whitelist/blacklist/b.txt
deleted file mode 100644
index 038d718..0000000
--- a/whitelist/blacklist/b.txt
+++ /dev/null
@@ -1 +0,0 @@
-testing
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
--- a/whitelist/blacklist/b.txt
+++ /dev/null
@@ -1 +0,0 @@
-testing
"""
assert file_dict['new_filename'] is None
assert parser.get_changed_files() == set(['whitelist/blacklist/b.txt'])
assert parser.get_lines_by_file() == {}
def test_dev_null_source(self):
patch = """diff --git a/whitelist/blacklist/a.txt b/whitelist/blacklist/a.txt
new file mode 100644
index 0000000..038d718
--- /dev/null
+++ b/whitelist/blacklist/a.txt
@@ -0,0 +1 @@
+testing
"""
parser = DiffParser(patch)
(file_dict,) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == """
--- /dev/null
+++ b/whitelist/blacklist/a.txt
@@ -0,0 +1 @@
+testing
"""
assert file_dict['old_filename'] is None
assert parser.get_changed_files() == set(['whitelist/blacklist/a.txt'])
assert parser.get_lines_by_file() == {'whitelist/blacklist/a.txt': {1}}
def process_diff(self):
lines_by_file = defaultdict(set)
try:
source = self.step.job.build.source
except AttributeError:
return lines_by_file
diff = source.generate_diff()
if not diff:
return lines_by_file
diff_parser = DiffParser(diff)
parsed_diff = diff_parser.parse()
for file_diff in parsed_diff:
for diff_chunk in file_diff['chunks']:
if not file_diff['new_filename']:
continue
lines_by_file[file_diff['new_filename'][2:]].update(
d['new_lineno'] for d in diff_chunk if d['action'] in ('add', 'del')
)
return lines_by_file
def process_diff(self):
lines_by_file = defaultdict(set)
try:
source = self.step.job.build.source
except AttributeError:
return lines_by_file
diff = source.generate_diff()
if not diff:
return lines_by_file
diff_parser = DiffParser(diff)
parsed_diff = diff_parser.parse()
for file_diff in parsed_diff:
for diff_chunk in file_diff['chunks']:
if not file_diff['new_filename']:
continue
lines_by_file[file_diff['new_filename'][2:]].update(
d['new_lineno'] for d in diff_chunk if d['action'] in ('add', 'del')
)
return lines_by_file
def test_remove_empty_file(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
deleted file mode 100644
index e69de29..0000000
"""
parser = DiffParser(patch)
(file_info, ) = parser.parse()
diff = parser.reconstruct_file_diff(file_info)
assert diff == ""
assert file_info.new_filename is None
assert parser.get_changed_files() == set(['diff-from/__init__.py'])
assert parser.get_lines_by_file() == {}
def test_add_empty_file(self):
patch = """diff --git a/diff-from/__init__.py b/diff-from/__init__.py
new file mode 100644
index 0000000..e69de29
"""
parser = DiffParser(patch)
(file_dict, ) = parser.parse()
diff = parser.reconstruct_file_diff(file_dict)
assert diff == ""
assert file_dict.old_filename is None
assert parser.get_changed_files() == set(['diff-from/__init__.py'])
assert parser.get_lines_by_file() == {}
def test_reconstruct_file_diff_complex_diff(self):
parser = DiffParser(COMPLEX_DIFF)
files = parser.parse()
diffs = set(parser.reconstruct_file_diff(x) for x in files)
assert len(diffs) == 3
correct = set([
"""
--- a/ci/run_with_retries.py
+++ b/ci/run_with_retries.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+import argparse
import os
import sys
import subprocess
@@ -41,7 +42,7 @@
return [testcase for testcase in root if testcase_status(testcase) in ('failure', 'error')]
""" + ' ' + """
""" + ' ' + """
-def run(files):
+def run(files, cwd):
cmd = COVERAGE_COMMAND_LINE % PYTEST_COMMAND_LINE
cmd = "%s %s" % (cmd % FINAL_JUNIT_XML_FILE, files)
write_out("Running command: %s" % cmd)
@@ -49,6 +50,16 @@
write_out("Generating coverage.xml")
run_streaming_out(COVERAGE_XML_COMMAND_LINE)
""" + ' ' + """
+ new_file_text = ""
+ if os.path.isfile('%s/coverage.xml' % os.getcwd()):
+ write_out("Replacing all paths in coverage.xml with repo paths.")
+ with open('%s/coverage.xml' % os.getcwd(), 'r') as f:
+ file_text = f.read()
+ new_file_text = file_text.replace("filename='", "filename='%s" % cwd)
+
+ with open('%s/coverage.xml' % os.getcwd(), 'w') as f:
+ f.write(new_file_text)
+
if junit_xml is None:
# rerun original command, hence rerunning all tests.
# this may be caused by a timeout.
@@ -171,5 +182,10 @@
if os.path.isfile(test_file):
subprocess.Popen("rm %s" % test_file)
""" + ' ' + """
- files_args = ' '.join(sys.argv[1:])
- run(files_args)
+ parser = argparse.ArgumentParser(description='Run the tests with retries')
+ parser.add_argument('filenames', metavar='filename', nargs='*', help="Files to run on")
+ parser.add_argument('--cwd', dest='cwd', help="path inside the repo to the cwd")
+
+ args = parser.parse_args()
+ files_args = ' '.join(args.filenames)
+ run(files_args, args.cwd)
""", """
--- a/ci/server-collect
+++ b/ci/server-collect
@@ -21,14 +21,14 @@
'name': 'blockserver',
'cwd': 'blockserver',
'path': 'blockserver',
- 'exec': pytest_command_line,
+ 'exec': pytest_command_line + ' --cwd blockserver/',
'xunit': 'tests.xml',
},
'metaserver': {
'name': 'metaserver',
'cwd': 'metaserver',
'path': 'metaserver',
- 'exec': pytest_command_line,
+ 'exec': pytest_command_line + ' --cwd metaserver/',
'xunit': 'tests.xml',
},
'dropbox': {
@@ -36,14 +36,14 @@
'cwd': 'dropbox_tests',
'path': 'dropbox/tests',
'keep_path': 1,
- 'exec': pytest_command_line,
+ 'exec': pytest_command_line + ' --cwd dropbox/',
'xunit': 'tests.xml',
},
'shortserver': {
'name': 'shortserver',
'cwd': 'shortserver',
'path': 'shortserver',
- 'exec': pytest_command_line,
+ 'exec': pytest_command_line + ' --cwd shortserver/',
'xunit': 'tests.xml',
},
}
""", """
--- a/ci/not-real
+++ b/ci/not-real
@@ -1 +1 @@
-Single Line
+Single Line!
"""
])
assert correct == diffs
def _selectively_apply_diff(self, file_path, file_content, diff):
"""A helper function that takes a diff, extract the parts of the diff
relating to `file_path`, and apply it to `file_content`.
If the diff does not involve `file_path`, then `file_content` is
returned, untouched.
Args:
file_path (str) - the path of the file to look for in the diff
file_content (str) - the content of the file to base on
diff (str) - diff in unidiff format
Returns:
str - `file_content` with the diff applied on top of it
Raises:
InvalidDiffError - when the supplied diff is invalid.
"""
parser = DiffParser(diff)
selected_diff = None
for file_dict in parser.parse():
if file_dict['new_filename'] is not None and file_dict['new_filename'][2:] == file_path:
selected_diff = parser.reconstruct_file_diff(file_dict)
if selected_diff is None:
return file_content
temp_patch_file_path = None
temp_dir = None
try:
# create a temporary file to house the patch
fd, temp_patch_file_path = tempfile.mkstemp()
os.write(fd, selected_diff)
os.close(fd)
# create a temporary folder where we will mimic the structure of
# the repo, with only the config inside it
dir_name, _ = os.path.split(file_path)
temp_dir = tempfile.mkdtemp()
if len(dir_name) > 0:
os.makedirs(os.path.join(temp_dir, dir_name))
temp_file_path = os.path.join(temp_dir, file_path)
with open(temp_file_path, 'w') as f:
f.write(file_content)
# apply the patch
try:
check_call([
'patch',
'--strip=1',
'--unified',
'--directory={}'.format(temp_dir),
'--input={}'.format(temp_patch_file_path),
])
except CalledProcessError:
raise InvalidDiffError
with open(temp_file_path, 'r') as f:
patched_content = f.read()
finally:
# clean up
if temp_patch_file_path and os.path.exists(temp_patch_file_path):
os.remove(temp_patch_file_path)
if temp_dir and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
return patched_content
def post(self):
"""
Create a new commit or diff build. The API roughly goes like this:
1. Identify the project(s) to build for. This can be done by specifying
``project``, ``repository``, or ``repository[callsign]``. If a repository is
specified somehow, then all projects for that repository are considered
for building.
2. Using the ``sha``, find the appropriate revision object. This may
involve updating the repo.
3. If ``patch`` is given, then apply the patch and mark this as a diff build.
Otherwise, this is a commit build.
4. If provided, apply project_whitelist, filtering out projects not in
this whitelist.
5. Based on the flag ``apply_project_files_trigger`` (see comment on the argument
itself for default values), decide whether or not to filter out projects
by file blacklist and whitelist.
6. Attach metadata and create/ensure existence of a build for each project,
depending on the flag ``ensure_only``.
NOTE: In ensure-only mode, the collection_ids of the returned builds are
not necessarily identical, as we give new builds new collection IDs
and preserve the existing builds' collection IDs.
NOTE: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch.
Not relevant until we fix TODO: ``sha`` is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if args.patch_file and args.ensure_only:
return error("Ensure-only mode does not work with a diff build yet.", problems=["patch", "ensure_only"])
if not (args.project or args.repository or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=["project", "repository",
"repository[phabricator.callsign]"])
# read arguments
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
# 1. identify project(s)
projects, repository = try_get_projects_and_repository(args)
if not projects:
return error("Unable to find project(s).")
# read arguments
label = args.label
author = args.author
message = args.message
tag = args.tag
if not tag and args.patch_file:
tag = 'patch'
# 2. find revision
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." % (
args.sha, repository.url), problems=['sha', 'repository'])
# get default values for arguments
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
# 3. Check for patch
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist'])
# mark as commit or diff build
if not patch:
is_commit_build = True
else:
is_commit_build = False
apply_project_files_trigger = args.apply_project_files_trigger
if apply_project_files_trigger is None:
apply_project_files_trigger = args.apply_file_whitelist
if apply_project_files_trigger is None:
if is_commit_build:
apply_project_files_trigger = False
else:
apply_project_files_trigger = True
if apply_project_files_trigger:
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
elif revision:
try:
files_changed = _get_revision_changed_files(repository, revision)
except MissingRevision:
return error("Unable to find commit %s in %s." % (
args.sha, repository.url), problems=['sha', 'repository'])
else:
# the only way that revision can be null is if this repo does not have a vcs backend
logging.warning('Revision and patch are both None for sha %s. This is because the repo %s does not have a VCS backend.', sha, repository.url)
files_changed = None
else:
# we won't be applying file whitelist, so there is no need to get the list of changed files.
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s', project.slug)
continue
# 4. apply project whitelist as appropriate
if args.project_whitelist is not None and project.slug not in args.project_whitelist:
logging.info('Project %s is not in the supplied whitelist', project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
# 5. apply file whitelist as appropriate
diff = None
if patch is not None:
diff = patch.diff
try:
if (
apply_project_files_trigger
and files_changed is not None
and not files_changed_should_trigger_project(
files_changed, project, project_options[project.id], sha, diff)
):
logging.info('Changed files do not trigger build for project %s', project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified.
pass
except ProjectConfigError:
author_name = '(Unknown)'
if author:
author_name = author.name
logging.error('Project config for project %s is not in a valid format. Author is %s.', project.slug, author_name, exc_info=True)
# 6. create/ensure build
if args.ensure_only:
potentials = list(Build.query.filter(
Build.project_id == project.id,
Build.source.has(revision_sha=sha, patch=patch),
).order_by(
Build.date_created.desc() # newest first
).limit(1))
if len(potentials) == 0:
builds.append(create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
))
else:
builds.append(potentials[0])
else:
builds.append(create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
))
return self.respond(builds)
def _selectively_apply_diff(self, file_path, file_content, diff):
"""A helper function that takes a diff, extract the parts of the diff
relating to `file_path`, and apply it to `file_content`.
If the diff does not involve `file_path`, then `file_content` is
returned, untouched.
Args:
file_path (str) - the path of the file to look for in the diff
file_content (str) - the content of the file to base on
diff (str) - diff in unidiff format
Returns:
str - `file_content` with the diff applied on top of it
Raises:
InvalidDiffError - when the supplied diff is invalid.
"""
parser = DiffParser(diff)
selected_diff = None
for file_dict in parser.parse():
if file_dict['new_filename'] is not None and file_dict[
'new_filename'][2:] == file_path:
selected_diff = parser.reconstruct_file_diff(file_dict)
if selected_diff is None:
return file_content
temp_patch_file_path = None
temp_dir = None
try:
# create a temporary file to house the patch
fd, temp_patch_file_path = tempfile.mkstemp()
os.write(fd, selected_diff)
os.close(fd)
# create a temporary folder where we will mimic the structure of
# the repo, with only the config inside it
dir_name, _ = os.path.split(file_path)
temp_dir = tempfile.mkdtemp()
if len(dir_name) > 0:
os.makedirs(os.path.join(temp_dir, dir_name))
temp_file_path = os.path.join(temp_dir, file_path)
with open(temp_file_path, 'w') as f:
f.write(file_content)
# apply the patch
try:
check_call([
'patch',
'--strip=1',
'--unified',
'--directory={}'.format(temp_dir),
'--input={}'.format(temp_patch_file_path),
])
except CalledProcessError:
raise InvalidDiffError
with open(temp_file_path, 'r') as f:
patched_content = f.read()
finally:
# clean up
if temp_patch_file_path and os.path.exists(temp_patch_file_path):
os.remove(temp_patch_file_path)
if temp_dir and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
return patched_content
def post(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
args = self.parser.parse_args()
repository = args.repository
if not args.repository:
return error("Repository not found")
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
projects = [p for p in projects if options.get(p.id, '1') == '1']
if not projects:
return self.respond([])
statsreporter.stats().incr('diffs_posted_from_phabricator')
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D{}'.format(args['phabricator.revisionID'])
try:
identify_revision(repository, sha)
except MissingRevision:
# This may just be a broken request (which is why we respond with a 400) but
# it also might indicate Phabricator and Changes being out of sync somehow,
# so we err on the side of caution and log it as an error.
logging.error(
"Diff %s was posted for an unknown revision (%s, %s)", target,
sha, repository.url)
return error("Unable to find commit %s in %s." %
(sha, repository.url),
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.error("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
try:
if not files_changed_should_trigger_project(
files_changed,
project,
project_options[project.id],
sha,
diff=patch.diff):
logging.info(
'No changed files matched project trigger for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified
pass
except ProjectConfigError:
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author.name,
exc_info=True)
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
tag="phabricator",
))
# This is the counterpoint to the above 'diffs_posted_from_phabricator';
# at this point we've successfully processed the diff, so comparing this
# stat to the above should give us the phabricator diff failure rate.
statsreporter.stats().incr(
'diffs_successfully_processed_from_phabricator')
return self.respond(builds)
def get(self, build_id):
build = Build.query.get(build_id)
if build is None:
return '', 404
args = self.parser.parse_args()
results = get_coverage_by_build_id(build.id)
if args.diff:
diff = build.source.generate_diff()
if not diff:
return self.respond({})
diff_parser = DiffParser(diff)
lines_by_file = diff_parser.get_lines_by_file()
results = [r for r in results if r.filename in lines_by_file]
coverage_data = merged_coverage_data(results)
coverage_stats = {}
for filename in lines_by_file:
if filename in coverage_data and filename in lines_by_file:
stats = get_coverage_stats(lines_by_file[filename],
coverage_data[filename])
coverage_stats[filename] = {
'linesCovered': stats.lines_covered,
'linesUncovered': stats.lines_uncovered,
'diffLinesCovered': stats.diff_lines_covered,
'diffLinesUncovered': stats.diff_lines_uncovered,
}
else:
# NOTE: Without a diff, the stats may be off if there are
# multiple job steps. (Each job step can potentially
# return a separate FileCoverage row for the same file.)
# For each file, we return the best metrics using
# min()/max(); if you want more correct metrics, pass
# diff=1.
coverage_stats = {}
for r in results:
if r.filename not in coverage_stats:
coverage_stats[r.filename] = {
'linesCovered': r.lines_covered,
'linesUncovered': r.lines_uncovered,
'diffLinesCovered': r.diff_lines_covered,
'diffLinesUncovered': r.diff_lines_uncovered,
}
else:
# Combine metrics using max() for [diff] lines
# covered, min() for [diff] lines uncovered.
stats = coverage_stats[r.filename]
coverage_stats[r.filename] = {
'linesCovered':
max(stats['linesCovered'], r.lines_covered),
'linesUncovered':
min(stats['linesUncovered'], r.lines_uncovered),
'diffLinesCovered':
max(stats['diffLinesCovered'], r.diff_lines_covered),
'diffLinesUncovered':
min(stats['diffLinesUncovered'],
r.diff_lines_uncovered),
}
return self.respond(coverage_stats)
def post(self):
"""
Create a new commit or diff build. The API roughly goes like this:
1. Identify the project(s) to build for. This can be done by specifying
``project``, ``repository``, or ``repository[callsign]``. If a repository is
specified somehow, then all projects for that repository are considered
for building.
2. Using the ``sha``, find the appropriate revision object. This may
involve updating the repo.
3. If ``patch`` is given, then apply the patch and mark this as a diff build.
Otherwise, this is a commit build.
4. If ``snapshot_id`` is given, verify that the snapshot can be used by all
projects.
5. If provided, apply project_whitelist, filtering out projects not in
this whitelist.
6. Based on the flag ``apply_project_files_trigger`` (see comment on the argument
itself for default values), decide whether or not to filter out projects
by file blacklist and whitelist.
7. Attach metadata and create/ensure existence of a build for each project,
depending on the flag ``ensure_only``.
NOTE: In ensure-only mode, the collection_ids of the returned builds are
not necessarily identical, as we give new builds new collection IDs
and preserve the existing builds' collection IDs.
NOTE: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch.
Not relevant until we fix TODO: ``sha`` is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if args.patch_file and args.ensure_only:
return error("Ensure-only mode does not work with a diff build yet.",
problems=["patch", "ensure_only"])
if not (args.project or args.repository or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=["project", "repository", "repository[phabricator.callsign]"])
# read arguments
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
# 1. identify project(s)
projects, repository = try_get_projects_and_repository(args)
if not projects:
return error("Unable to find project(s).")
# read arguments
label = args.label
author = args.author
message = args.message
tag = args.tag
snapshot_id = args.snapshot_id
no_snapshot = args.no_snapshot
cause = Cause[args.cause]
if no_snapshot and snapshot_id:
return error("Cannot specify snapshot with no_snapshot option")
if not tag and args.patch_file:
tag = 'patch'
# 2. validate snapshot
if snapshot_id:
snapshot = Snapshot.query.get(snapshot_id)
if not snapshot:
return error("Unable to find snapshot.")
if snapshot.status != SnapshotStatus.active:
return error("Snapshot is in an invalid state: %s" % snapshot.status)
for project in projects:
plans = get_build_plans(project)
for plan in plans:
plan_options = plan.get_item_options()
allow_snapshot = '1' == plan_options.get('snapshot.allow', '1') or plan.snapshot_plan
if allow_snapshot and not SnapshotImage.get(plan, snapshot_id):
# We want to create a build using a specific snapshot but no image
# was found for this plan so fail.
return error("Snapshot cannot be applied to %s's %s" % (project.slug, plan.label))
# 3. find revision
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." % (args.sha, repository.url),
problems=['sha', 'repository'])
# get default values for arguments
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
# 4. Check for patch
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(projects, ['build.file-whitelist'])
# mark as commit or diff build
if not patch:
is_commit_build = True
else:
is_commit_build = False
apply_project_files_trigger = args.apply_project_files_trigger
if apply_project_files_trigger is None:
apply_project_files_trigger = args.apply_file_whitelist
if apply_project_files_trigger is None:
if is_commit_build:
apply_project_files_trigger = False
else:
apply_project_files_trigger = True
if apply_project_files_trigger:
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
elif revision:
try:
files_changed = _get_revision_changed_files(repository, revision)
except MissingRevision:
return error("Unable to find commit %s in %s." % (args.sha, repository.url),
problems=['sha', 'repository'])
else:
# the only way that revision can be null is if this repo does not have a vcs backend
logging.warning('Revision and patch are both None for sha %s. This is because the repo %s does not have a VCS backend.', sha, repository.url)
files_changed = None
else:
# we won't be applying file whitelist, so there is no need to get the list of changed files.
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s', project.slug)
continue
# 5. apply project whitelist as appropriate
if args.project_whitelist is not None and project.slug not in args.project_whitelist:
logging.info('Project %s is not in the supplied whitelist', project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
# 6. apply file whitelist as appropriate
diff = None
if patch is not None:
diff = patch.diff
if (
apply_project_files_trigger and
files_changed is not None and
not files_changed_should_trigger_project(
files_changed, project, project_options[project.id], sha, diff)
):
logging.info('Changed files do not trigger build for project %s', project.slug)
continue
# 7. create/ensure build
build_message = None
selective_testing_policy = SelectiveTestingPolicy.disabled
if args.selective_testing and project_lib.contains_active_autogenerated_plan(project):
if is_commit_build:
selective_testing_policy, reasons = get_selective_testing_policy(project, sha, diff)
if reasons:
if selective_testing_policy is SelectiveTestingPolicy.disabled:
reasons = ["Selective testing was requested but not done because:"] + [' ' + m for m in reasons]
build_message = '\n'.join(reasons)
else:
# NOTE: for diff builds, it makes sense to just do selective testing,
# since it will never become a parent build and will never be used to
# calculate revision results.
selective_testing_policy = SelectiveTestingPolicy.enabled
if args.ensure_only:
potentials = list(Build.query.filter(
Build.project_id == project.id,
Build.source.has(revision_sha=sha, patch=patch),
).order_by(
Build.date_created.desc() # newest first
).limit(1))
if len(potentials) == 0:
builds.append(create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
cause=cause,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot,
selective_testing_policy=selective_testing_policy,
))
else:
builds.append(potentials[0])
else:
builds.append(create_build(
project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
cause=cause,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot,
selective_testing_policy=selective_testing_policy,
))
if build_message:
message = BuildMessage(
build=builds[-1],
text=build_message,
)
db.session.add(message)
db.session.commit()
return self.respond(builds)
def test_parse_simple_diff(self):
parser = DiffParser(SIMPLE_DIFF)
files = parser.parse()
assert files == [
{
'is_header': False,
'old_filename': 'a/changes/utils/diff_parser.py',
'old_revision': None,
'new_filename': 'b/changes/utils/diff_parser.py',
'new_revision': None,
'chunk_markers': ['@@ -71,6 +71,7 @@ class DiffParser(object):'],
'chunks': [[
{
'action': 'unmod',
'line': '',
'new_lineno': 71,
'old_lineno': 71,
'ends_with_newline': True,
},
{
'action': 'unmod',
'line': ' in_header = False',
'new_lineno': 72,
'old_lineno': 72,
'ends_with_newline': True,
},
{
'action': 'unmod',
'line': ' chunks = []',
'new_lineno': 73,
'old_lineno': 73,
'ends_with_newline': True,
},
{
'action': 'add',
'line': ' chunk_markers = []',
'new_lineno': 74,
'old_lineno': u'',
'ends_with_newline': True,
},
{
'action': 'unmod',
'line': ' old, new = self._extract_rev(line, lineiter.next())',
'new_lineno': 75,
'old_lineno': 74,
'ends_with_newline': True,
},
{
'action': 'unmod',
'line': ' files.append({',
'new_lineno': 76,
'old_lineno': 75,
'ends_with_newline': True,
},
{
'action': 'unmod',
'line': " 'is_header': False,",
'new_lineno': 77,
'old_lineno': 76,
'ends_with_newline': True,
}
]],
}
]
def post(self):
"""
Create a new commit or diff build. The API roughly goes like this:
1. Identify the project(s) to build for. This can be done by specifying
``project``, ``repository``, or ``repository[callsign]``. If a repository is
specified somehow, then all projects for that repository are considered
for building.
2. Using the ``sha``, find the appropriate revision object. This may
involve updating the repo.
3. If ``patch`` is given, then apply the patch and mark this as a diff build.
Otherwise, this is a commit build.
4. If ``snapshot_id`` is given, verify that the snapshot can be used by all
projects.
5. If provided, apply project_whitelist, filtering out projects not in
this whitelist.
6. Based on the flag ``apply_project_files_trigger`` (see comment on the argument
itself for default values), decide whether or not to filter out projects
by file blacklist and whitelist.
7. Attach metadata and create/ensure existence of a build for each project,
depending on the flag ``ensure_only``.
NOTE: In ensure-only mode, the collection_ids of the returned builds are
not necessarily identical, as we give new builds new collection IDs
and preserve the existing builds' collection IDs.
NOTE: If ``patch`` is specified ``sha`` is assumed to be the original
base revision to apply the patch.
Not relevant until we fix TODO: ``sha`` is **not** guaranteed to be the rev
used to apply the patch. See ``find_green_parent_sha`` for the logic of
identifying the correct revision.
"""
args = self.parser.parse_args()
if args.patch_file and args.ensure_only:
return error(
"Ensure-only mode does not work with a diff build yet.",
problems=["patch", "ensure_only"])
if not (args.project or args.repository
or args['repository[phabricator.callsign]']):
return error("Project or repository must be specified",
problems=[
"project", "repository",
"repository[phabricator.callsign]"
])
# read arguments
if args.patch_data:
try:
patch_data = json.loads(args.patch_data)
except Exception:
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
if not isinstance(patch_data, dict):
return error("Invalid patch data (must be JSON dict)",
problems=["patch[data]"])
else:
patch_data = None
# 1. identify project(s)
projects, repository = try_get_projects_and_repository(args)
if not projects:
return error("Unable to find project(s).")
# read arguments
label = args.label
author = args.author
message = args.message
tag = args.tag
snapshot_id = args.snapshot_id
no_snapshot = args.no_snapshot
if no_snapshot and snapshot_id:
return error("Cannot specify snapshot with no_snapshot option")
if not tag and args.patch_file:
tag = 'patch'
# 2. validate snapshot
if snapshot_id:
snapshot = Snapshot.query.get(snapshot_id)
if not snapshot:
return error("Unable to find snapshot.")
if snapshot.status != SnapshotStatus.active:
return error("Snapshot is in an invalid state: %s" %
snapshot.status)
for project in projects:
plans = get_build_plans(project)
for plan in plans:
plan_options = plan.get_item_options()
allow_snapshot = '1' == plan_options.get(
'snapshot.allow', '0') or plan.snapshot_plan
if allow_snapshot and not SnapshotImage.get(
plan, snapshot_id):
# We want to create a build using a specific snapshot but no image
# was found for this plan so fail.
return error("Snapshot cannot be applied to %s's %s" %
(project.slug, plan.label))
# 3. find revision
try:
revision = identify_revision(repository, args.sha)
except MissingRevision:
# if the default fails, we absolutely can't continue and the
# client should send a valid revision
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
# get default values for arguments
if revision:
if not author:
author = revision.author
if not label:
label = revision.subject
# only default the message if its absolutely not set
if message is None:
message = revision.message
sha = revision.sha
else:
sha = args.sha
if not args.target:
target = sha[:12]
else:
target = args.target[:128]
if not label:
if message:
label = message.splitlines()[0]
if not label:
label = 'A homeless build'
label = label[:128]
# 4. Check for patch
if args.patch_file:
fp = StringIO()
for line in args.patch_file:
fp.write(line)
patch_file = fp
else:
patch_file = None
if patch_file:
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=patch_file.getvalue(),
)
db.session.add(patch)
else:
patch = None
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
# mark as commit or diff build
if not patch:
is_commit_build = True
else:
is_commit_build = False
apply_project_files_trigger = args.apply_project_files_trigger
if apply_project_files_trigger is None:
apply_project_files_trigger = args.apply_file_whitelist
if apply_project_files_trigger is None:
if is_commit_build:
apply_project_files_trigger = False
else:
apply_project_files_trigger = True
if apply_project_files_trigger:
if patch:
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
elif revision:
try:
files_changed = _get_revision_changed_files(
repository, revision)
except MissingRevision:
return error("Unable to find commit %s in %s." %
(args.sha, repository.url),
problems=['sha', 'repository'])
else:
# the only way that revision can be null is if this repo does not have a vcs backend
logging.warning(
'Revision and patch are both None for sha %s. This is because the repo %s does not have a VCS backend.',
sha, repository.url)
files_changed = None
else:
# we won't be applying file whitelist, so there is no need to get the list of changed files.
files_changed = None
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
# 5. apply project whitelist as appropriate
if args.project_whitelist is not None and project.slug not in args.project_whitelist:
logging.info('Project %s is not in the supplied whitelist',
project.slug)
continue
forced_sha = sha
# TODO(dcramer): find_green_parent_sha needs to take branch
# into account
# if patch_file:
# forced_sha = find_green_parent_sha(
# project=project,
# sha=sha,
# )
# 6. apply file whitelist as appropriate
diff = None
if patch is not None:
diff = patch.diff
try:
if (apply_project_files_trigger and files_changed is not None
and not files_changed_should_trigger_project(
files_changed, project,
project_options[project.id], sha, diff)):
logging.info(
'Changed files do not trigger build for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified.
pass
except ProjectConfigError:
author_name = '(Unknown)'
if author:
author_name = author.name
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author_name,
exc_info=True)
# 7. create/ensure build
if args.ensure_only:
potentials = list(
Build.query.filter(
Build.project_id == project.id,
Build.source.has(revision_sha=sha, patch=patch),
).order_by(Build.date_created.desc() # newest first
).limit(1))
if len(potentials) == 0:
builds.append(
create_build(project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot))
else:
builds.append(potentials[0])
else:
builds.append(
create_build(project=project,
collection_id=collection_id,
sha=forced_sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
source_data=patch_data,
tag=tag,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot))
return self.respond(builds)
def test_parse_complex_diff(self):
parser = DiffParser(COMPLEX_DIFF)
files = parser.parse()
assert len(files) == 3
def post(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
args = self.parser.parse_args()
repository = args.repository
if not args.repository:
return error("Repository not found")
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
projects = [p for p in projects if options.get(p.id, '1') == '1']
if not projects:
return self.respond([])
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D{}'.format(args['phabricator.revisionID'])
try:
identify_revision(repository, sha)
except MissingRevision:
return error("Unable to find commit %s in %s." %
(sha, repository.url),
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.error("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
if not in_project_files_whitelist(project_options[project.id],
files_changed):
logging.info(
'No changed files matched build.file-whitelist for project %s',
project.slug)
continue
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
))
return self.respond(builds)
def post_impl(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
# we manually check for arg presence here so we can send a more specific
# error message to the user (rather than a plain 400)
args = self.parser.parse_args()
if not args.repository:
# No need to postback a comment for this
statsreporter.stats().incr("diffs_repository_not_found")
return error("Repository not found")
repository = args.repository
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
# Filter out projects that aren't configured to run builds off of diffs
# - Diff trigger disabled
# - No build plans
projects = [
p for p in projects
if options.get(p.id, '1') == '1' and get_build_plans(p)
]
if not projects:
return self.respond([])
statsreporter.stats().incr('diffs_posted_from_phabricator')
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D%s' % args['phabricator.revisionID']
try:
identify_revision(repository, sha)
except MissingRevision:
# This may just be a broken request (which is why we respond with a 400) but
# it also might indicate Phabricator and Changes being out of sync somehow,
# so we err on the side of caution and log it as an error.
logging.error(
"Diff %s was posted for an unknown revision (%s, %s)", target,
sha, repository.url)
# We should postback since this can happen if a user diffs dependent revisions
statsreporter.stats().incr("diffs_missing_base_revision")
return self.postback_error(
"Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n"
" - You may be working on multiple stacked diffs in your local repository.\n"
" {revision} only exists in your local copy. Changes thus cannot apply your patch\n"
" - If you are sure that's not the case, it's possible you applied your patch to an extremely\n"
" recent revision which Changes hasn't picked up yet. Retry in a minute\n"
.format(
revision=sha,
repo=repository.url,
),
target,
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(line.decode('utf-8') for line in args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.warning("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
# No need to inform user about this explicitly
statsreporter.stats().incr("diffs_already_exists")
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
# We already filtered out empty build plans
assert plan_list, ('No plans defined for project {}'.format(
project.slug))
try:
if not files_changed_should_trigger_project(
files_changed,
project,
project_options[project.id],
sha,
diff=patch.diff):
logging.info(
'No changed files matched project trigger for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified
pass
except ProjectConfigError:
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author.name,
exc_info=True)
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
tag="phabricator",
))
# This is the counterpoint to the above 'diffs_posted_from_phabricator';
# at this point we've successfully processed the diff, so comparing this
# stat to the above should give us the phabricator diff failure rate.
statsreporter.stats().incr(
'diffs_successfully_processed_from_phabricator')
return self.respond(builds)
