def get_chunks_uncached(self):
"""Yield the list of chunks, bypassing the cache."""
old = get_original_file(self.filediff, self.request, self.encoding_list)
new = get_patched_file(old, self.filediff, self.request)
if self.filediff.orig_sha1 is None:
self.filediff.extra_data.update(
{"orig_sha1": self._get_checksum(old), "patched_sha1": self._get_checksum(new)}
)
self.filediff.save(update_fields=["extra_data"])
if self.interfilediff:
old = new
interdiff_orig = get_original_file(self.interfilediff, self.request, self.encoding_list)
new = get_patched_file(interdiff_orig, self.interfilediff, self.request)
if self.interfilediff.orig_sha1 is None:
self.interfilediff.extra_data.update(
{"orig_sha1": self._get_checksum(interdiff_orig), "patched_sha1": self._get_checksum(new)}
)
self.interfilediff.save(update_fields=["extra_data"])
elif self.force_interdiff:
# Basically, revert the change.
old, new = new, old
if self.interfilediff:
log_timer = log_timed(
"Generating diff chunks for interdiff ids %s-%s (%s)"
% (self.filediff.id, self.interfilediff.id, self.filediff.source_file),
request=self.request,
)
else:
log_timer = log_timed(
"Generating diff chunks for self.filediff id %s (%s)" % (self.filediff.id, self.filediff.source_file),
request=self.request,
)
for chunk in self.generate_chunks(old, new):
yield chunk
log_timer.done()
if not self.interfilediff and not self.force_interdiff:
insert_count = self.counts["insert"]
delete_count = self.counts["delete"]
replace_count = self.counts["replace"]
equal_count = self.counts["equal"]
self.filediff.set_line_counts(
insert_count=insert_count,
delete_count=delete_count,
replace_count=replace_count,
equal_count=equal_count,
total_line_count=(insert_count + delete_count + replace_count + equal_count),
)
def get(self, request, diffset_id=None, *args, **kwargs):
"""Returns the original unpatched file.
The file is returned as :mimetype:`text/plain` and is the original
file before applying a patch.
"""
try:
attached_diffset = DiffSet.objects.filter(pk=diffset_id, history__isnull=True)
if attached_diffset.exists():
filediff_resource = resources.filediff
else:
filediff_resource = resources.draft_filediff
filediff = filediff_resource.get_object(request, diffset=diffset_id, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if filediff.is_new:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(filediff, request, filediff.diffset.repository.get_encoding_list())
except Exception as e:
logging.error("Error retrieving original file: %s", e, exc_info=1, request=request)
return FILE_RETRIEVAL_ERROR
resp = HttpResponse(orig_file, mimetype="text/plain")
filename = urllib_quote(filediff.source_file)
resp["Content-Disposition"] = "inline; filename=%s" % filename
set_last_modified(resp, filediff.diffset.timestamp)
return resp
def get(self, request, diffset_id=None, *args, **kwargs):
"""Returns the patched file.
The file is returned as :mimetype:`text/plain` and is the result
of applying the patch to the original file.
"""
try:
attached_diffset = DiffSet.objects.filter(pk=diffset_id,
history__isnull=True)
if attached_diffset.exists():
filediff_resource = resources.filediff
else:
filediff_resource = resources.draft_filediff
filediff = filediff_resource.get_object(
request, diffset=diffset_id, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if filediff.deleted:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(filediff, request=request)
except Exception, e:
logging.error("Error retrieving original file: %s", e, exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
def get(self, request, *args, **kwargs):
"""Returns the original file.
The file is returned as :mimetype:`text/plain` and is the original
file before applying a patch.
"""
result = self.get_filediff(request, *args, **kwargs)
if isinstance(result, FileDiff):
filediff = result
elif isinstance(result, WebAPIError):
return result
else:
raise ValueError('Unexpected result from get_filediff')
if filediff.is_new:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(
filediff, request,
filediff.diffset.repository.get_encoding_list())
except Exception as e:
logging.error('%s: Error retrieving original file for FileDiff '
'%s: %s',
self.__class__.__name__, filediff.pk, e, exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
resp = HttpResponse(orig_file, content_type='text/plain')
filename = urllib_quote(filediff.source_file)
resp['Content-Disposition'] = 'inline; filename=%s' % filename
set_last_modified(resp, filediff.diffset.timestamp)
return resp
def _download_analysis(analyze_function, request, review_request_id, revision,
filediff_id, local_site=None, modified=True):
"""Generates file analysis given by analyze_function on the specified file.
This will download the file as a string, write it to a temporary file
in the homefolder, run the analysis, delete the temporary file, and
output the filename and data_analysis
"""
logging.debug("Analyze_function" + analyze_function.__name__)
review_request, response = \
_find_review_request(request, review_request_id, local_site)
if not review_request:
return response
draft = review_request.get_draft(request.user)
diffset = _query_for_diff(review_request, request.user, revision, draft)
filediff = get_object_or_404(diffset.files, pk=filediff_id)
encoding_list = diffset.repository.get_encoding_list()
# Get a file either from the cache or the SCM, applying the parent diff if it exists.
# SCM exceptions are passed back to the caller.
working_file = get_original_file(filediff, request, encoding_list)
if modified:
working_file = get_patched_file(working_file, filediff, request)
working_file = convert_to_unicode(working_file, encoding_list)[1]
logging.debug("Encoding List: %s", encoding_list)
logging.debug("Source File: " + filediff.source_file)
temp_file_name = "cctempfile_" + filediff.source_file.replace("/","_")
logging.debug("temp_file_name: " + temp_file_name)
source_file = os.path.join(HOMEFOLDER, temp_file_name)
#with open (source_file, "r") as myfile:
# logging.debug("String: " + myfile.read().replace('\n', ''))
logging.debug("File contents" + working_file)
#temp_file = open(source_file, 'w')
#temp_file = codecs.open(source_file, encoding='utf-8')
#temp_file.write(working_file.encode('utf-8'))
temp_file = codecs.open(source_file, 'w', encoding='utf-8')
temp_file.write(working_file)
temp_file.close()
data_analysis = analyze_function(source_file)
os.remove(source_file)
if not data_analysis:
data_analysis = None
return filediff.source_file, data_analysis
def get(self, request, *args, **kwargs):
"""Returns the patched file.
The file is returned as :mimetype:`text/plain` and is the result
of applying the patch to the original file.
"""
result = self.get_filediff(request, *args, **kwargs)
if isinstance(result, FileDiff):
filediff = result
elif isinstance(result, WebAPIError):
return result
else:
raise ValueError('Unexpected result from get_filediff')
if filediff.deleted:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(filediff=filediff, request=request)
except Exception as e:
logging.error(
'%s: Error retrieving original file for FileDiff '
'%s: %s',
self.__class__.__name__,
filediff.pk,
e,
exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
try:
patched_file = get_patched_file(source_data=orig_file,
filediff=filediff,
request=request)
except Exception as e:
logging.error(
'%s: Error retrieving patched file for FileDiff '
'%s: %s',
self.__class__.__name__,
filediff.pk,
e,
exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
resp = HttpResponse(patched_file, content_type='text/plain')
filename = urllib_quote(filediff.dest_file)
resp['Content-Disposition'] = 'inline; filename=%s' % filename
set_last_modified(resp, filediff.diffset.timestamp)
return resp
def get(self, request, diffset_id=None, *args, **kwargs):
"""Returns the patched file.
The file is returned as :mimetype:`text/plain` and is the result
of applying the patch to the original file.
"""
try:
attached_diffset = DiffSet.objects.filter(pk=diffset_id,
history__isnull=True)
if attached_diffset.exists():
filediff_resource = resources.filediff
else:
filediff_resource = resources.draft_filediff
filediff = filediff_resource.get_object(request,
diffset=diffset_id,
*args,
**kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if filediff.deleted:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(
filediff, request,
filediff.diffset.repository.get_encoding_list())
except Exception as e:
logging.error("Error retrieving original file: %s",
e,
exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
try:
patched_file = get_patched_file(orig_file, filediff, request)
except Exception as e:
logging.error("Error retrieving patched file: %s",
e,
exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
resp = HttpResponse(patched_file, mimetype='text/plain')
filename = urllib_quote(filediff.dest_file)
resp['Content-Disposition'] = 'inline; filename=%s' % filename
set_last_modified(resp, filediff.diffset.timestamp)
return resp
def get(self, request, *args, **kwargs):
"""Returns the patched file.
The file is returned as :mimetype:`text/plain` and is the result
of applying the patch to the original file.
"""
result = self.get_filediff(request, *args, **kwargs)
if isinstance(result, FileDiff):
filediff = result
elif isinstance(result, WebAPIError):
return result
else:
raise ValueError("Unexpected result from get_filediff")
if filediff.deleted:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(filediff, request, filediff.diffset.repository.get_encoding_list())
except Exception as e:
logging.error(
"%s: Error retrieving original file for FileDiff " "%s: %s",
self.__class__.__name__,
filediff.pk,
e,
exc_info=1,
request=request,
)
return FILE_RETRIEVAL_ERROR
try:
patched_file = get_patched_file(orig_file, filediff, request)
except Exception as e:
logging.error(
"%s: Error retrieving patched file for FileDiff %" "%s: %s",
self.__class__.__name__,
filediff.pk,
e,
exc_info=1,
request=request,
)
return FILE_RETRIEVAL_ERROR
resp = HttpResponse(patched_file, content_type="text/plain")
filename = urllib_quote(filediff.dest_file)
resp["Content-Disposition"] = "inline; filename=%s" % filename
set_last_modified(resp, filediff.diffset.timestamp)
return resp
def get(self, request, diff_revision=None, *args, **kwargs):
"""Returns the patched file.
The file is returned as :mimetype:`text/plain` and is the result
of applying the patch to the original file.
"""
try:
attached_diffset = DiffSet.objects.filter(revision=diff_revision,
history__isnull=True)
if attached_diffset.exists():
filediff_resource = resources.filediff
else:
filediff_resource = resources.draft_filediff
filediff = filediff_resource.get_object(
request, diff_revision=diff_revision, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if filediff.deleted:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(
filediff, request,
filediff.diffset.repository.get_encoding_list())
except Exception as e:
logging.error("Error retrieving original file: %s", e, exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
try:
patched_file = get_patched_file(orig_file, filediff, request)
except Exception as e:
logging.error("Error retrieving patched file: %s", e, exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
resp = HttpResponse(patched_file, mimetype='text/plain')
filename = urllib_quote(filediff.dest_file)
resp['Content-Disposition'] = 'inline; filename=%s' % filename
set_last_modified(resp, filediff.diffset.timestamp)
return resp
def _download_analysis(analyze_function,
request,
review_request_id,
revision,
filediff_id,
local_site=None,
modified=True):
"""Generates file analysis given by analyze_function on the specified file.
This will download the file as a string, write it to a temporary file
in the homefolder, run the analysis, delete the temporary file, and
output the filename and data_analysis
"""
review_request, response = \
_find_review_request(request, review_request_id, local_site)
if not review_request:
return response
draft = review_request.get_draft(request.user)
diffset = _query_for_diff(review_request, request.user, revision, draft)
filediff = get_object_or_404(diffset.files, pk=filediff_id)
encoding_list = diffset.repository.get_encoding_list()
data = get_original_file(filediff, request, encoding_list)
if modified:
data = get_patched_file(data, filediff, request)
data = convert_to_unicode(data, encoding_list)[1]
temp_file_name = "cctempfile_" + filediff.source_file
source_file = os.path.join(HOMEFOLDER, temp_file_name)
temp_file = open(source_file, 'w')
temp_file.write(data)
temp_file.close()
data_analysis = analyze_function(source_file)
os.remove(source_file)
if not data_analysis:
data_analysis = None
return filediff.source_file, data_analysis
def get(self, request, *args, **kwargs):
"""Returns the patched file.
The file is returned as :mimetype:`text/plain` and is the result
of applying the patch to the original file.
"""
try:
filediff = resources.filediff.get_object(request, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if filediff.deleted:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(filediff, request=request)
except Exception, e:
logging.error("Error retrieving original file: %s", e, exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
def get(self, request, *args, **kwargs):
"""Returns the original unpatched file.
The file is returned as :mimetype:`text/plain` and is the original
file before applying a patch.
"""
try:
filediff = resources.filediff.get_object(request, *args, **kwargs)
except ObjectDoesNotExist:
return DOES_NOT_EXIST
if filediff.is_new:
return DOES_NOT_EXIST
try:
orig_file = get_original_file(filediff, request=request)
except Exception, e:
logging.error("Error retrieving original file: %s",
e,
exc_info=1,
request=request)
return FILE_RETRIEVAL_ERROR
def _get_chunks_uncached(self):
"""Returns the list of chunks, bypassing the cache."""
old = get_original_file(self.filediff, self.request)
new = get_patched_file(old, self.filediff, self.request)
if self.interfilediff:
old = new
interdiff_orig = get_original_file(self.interfilediff,
self.request)
new = get_patched_file(interdiff_orig, self.interfilediff,
self.request)
elif self.force_interdiff:
# Basically, revert the change.
old, new = new, old
encoding = self.diffset.repository.encoding or 'iso-8859-15'
old = self._convert_to_utf8(old, encoding)
new = self._convert_to_utf8(new, encoding)
# Normalize the input so that if there isn't a trailing newline, we add
# it.
if old and old[-1] != '\n':
old += '\n'
if new and new[-1] != '\n':
new += '\n'
a = self.NEWLINES_RE.split(old or '')
b = self.NEWLINES_RE.split(new or '')
# Remove the trailing newline, now that we've split this. This will
# prevent a duplicate line number at the end of the diff.
del a[-1]
del b[-1]
a_num_lines = len(a)
b_num_lines = len(b)
markup_a = markup_b = None
if self._get_enable_syntax_highlighting(old, new, a, b):
repository = self.filediff.diffset.repository
tool = repository.get_scmtool()
source_file = \
tool.normalize_path_for_display(self.filediff.source_file)
dest_file = \
tool.normalize_path_for_display(self.filediff.dest_file)
try:
# TODO: Try to figure out the right lexer for these files
# once instead of twice.
markup_a = self._apply_pygments(old or '', source_file)
markup_b = self._apply_pygments(new or '', dest_file)
except:
pass
if not markup_a:
markup_a = self.NEWLINES_RE.split(escape(old))
if not markup_b:
markup_b = self.NEWLINES_RE.split(escape(new))
siteconfig = SiteConfiguration.objects.get_current()
ignore_space = True
for pattern in siteconfig.get('diffviewer_include_space_patterns'):
if fnmatch.fnmatch(self.filename, pattern):
ignore_space = False
break
self.differ = get_differ(a,
b,
ignore_space=ignore_space,
compat_version=self.diffset.diffcompat)
self.differ.add_interesting_lines_for_headers(self.filename)
context_num_lines = siteconfig.get("diffviewer_context_num_lines")
collapse_threshold = 2 * context_num_lines + 3
if self.interfilediff:
log_timer = log_timed(
"Generating diff chunks for interdiff ids %s-%s (%s)" %
(self.filediff.id, self.interfilediff.id,
self.filediff.source_file),
request=self.request)
else:
log_timer = log_timed(
"Generating diff chunks for self.filediff id %s (%s)" %
(self.filediff.id, self.filediff.source_file),
request=self.request)
line_num = 1
opcodes_generator = get_diff_opcode_generator(self.differ,
self.filediff,
self.interfilediff)
for tag, i1, i2, j1, j2, meta in opcodes_generator:
old_lines = markup_a[i1:i2]
new_lines = markup_b[j1:j2]
num_lines = max(len(old_lines), len(new_lines))
self._cur_meta = meta
lines = map(self._diff_line, xrange(line_num,
line_num + num_lines),
xrange(i1 + 1, i2 + 1), xrange(j1 + 1, j2 + 1),
a[i1:i2], b[j1:j2], old_lines, new_lines)
self._cur_meta = None
if tag == 'equal' and num_lines > collapse_threshold:
last_range_start = num_lines - context_num_lines
if line_num == 1:
yield self._new_chunk(lines, 0, last_range_start, True)
yield self._new_chunk(lines, last_range_start, num_lines)
else:
yield self._new_chunk(lines, 0, context_num_lines)
if i2 == a_num_lines and j2 == b_num_lines:
yield self._new_chunk(lines, context_num_lines,
num_lines, True)
else:
yield self._new_chunk(lines, context_num_lines,
last_range_start, True)
yield self._new_chunk(lines, last_range_start,
num_lines)
else:
yield self._new_chunk(lines, 0, num_lines, False, tag, meta)
line_num += num_lines
log_timer.done()
def get_chunks_uncached(self):
"""Yield the list of chunks, bypassing the cache."""
old = get_original_file(self.filediff, self.request,
self.encoding_list)
new = get_patched_file(old, self.filediff, self.request)
if self.filediff.orig_sha1 is None:
self.filediff.extra_data.update({
'orig_sha1':
self._get_checksum(old),
'patched_sha1':
self._get_checksum(new),
})
self.filediff.save(update_fields=['extra_data'])
if self.interfilediff:
old = new
interdiff_orig = get_original_file(self.interfilediff,
self.request,
self.encoding_list)
new = get_patched_file(interdiff_orig, self.interfilediff,
self.request)
if self.interfilediff.orig_sha1 is None:
self.interfilediff.extra_data.update({
'orig_sha1':
self._get_checksum(interdiff_orig),
'patched_sha1':
self._get_checksum(new),
})
self.interfilediff.save(update_fields=['extra_data'])
elif self.force_interdiff:
# Basically, revert the change.
old, new = new, old
if self.interfilediff:
log_timer = log_timed(
"Generating diff chunks for interdiff ids %s-%s (%s)" %
(self.filediff.id, self.interfilediff.id,
self.filediff.source_file),
request=self.request)
else:
log_timer = log_timed(
"Generating diff chunks for self.filediff id %s (%s)" %
(self.filediff.id, self.filediff.source_file),
request=self.request)
for chunk in self.generate_chunks(old, new):
yield chunk
log_timer.done()
if not self.interfilediff:
insert_count = self.counts['insert']
delete_count = self.counts['delete']
replace_count = self.counts['replace']
equal_count = self.counts['equal']
self.filediff.set_line_counts(
insert_count=insert_count,
delete_count=delete_count,
replace_count=replace_count,
equal_count=equal_count,
total_line_count=(insert_count + delete_count + replace_count +
equal_count))
def test_create_with_parent_filediff_with_move_and_change(self):
"""Testing UploadDiffForm.create with a parent diff consisting of a
move/rename with content change
"""
revisions = [
b'93e6b3e8944c48737cb11a1e52b046fa30aea7a9',
b'4839fc480f47ca59cf05a9c39410ea744d1e17a2',
b'04861c126cfebd7e7cb93045ab0bff4a7acc4cf2',
]
parent_diff = SimpleUploadedFile('parent_diff',
(b'diff --git a/foo b/bar\n'
b'similarity index 55%%\n'
b'rename from foo\n'
b'rename to bar\n'
b'index %s..%s 100644\n'
b'--- a/foo\n'
b'+++ b/bar\n'
b'@@ -1,2 +1,3 @@\n'
b' Foo\n'
b'+Bar\n') %
(revisions[0], revisions[1]),
content_type='text/x-patch')
diff = SimpleUploadedFile('diff',
(b'diff --git a/bar b/bar\n'
b'index %s..%s 100644\n'
b'--- a/bar\n'
b'+++ b/bar\n'
b'@@ -1,3 +1,4 @@\n'
b' Foo\n'
b' Bar\n'
b'+Baz\n') % (revisions[1], revisions[2]),
content_type='text/x-patch')
repository = self.create_repository(tool_name='Test')
self.spy_on(repository.get_file_exists,
call_fake=lambda *args, **kwargs: True)
# We will only be making one call to get_file and we can fake it out.
self.spy_on(repository.get_file,
call_fake=lambda *args, **kwargs: b'Foo\n')
self.spy_on(patch)
form = UploadDiffForm(repository=repository,
data={
'basedir': '/',
},
files={
'path': diff,
'parent_diff_path': parent_diff,
})
self.assertTrue(form.is_valid())
diffset = form.create()
self.assertEqual(diffset.files.count(), 1)
f = diffset.files.get()
self.assertEqual(f.source_revision, revisions[0])
self.assertEqual(f.dest_detail, revisions[2])
original_file = get_original_file(f, None, ['ascii'])
self.assertEqual(original_file, b'Foo\nBar\n')
self.assertTrue(patch.spy.called)
patched_file = get_patched_file(original_file, f, None)
self.assertEqual(patched_file, b'Foo\nBar\nBaz\n')
self.assertEqual(len(patch.spy.calls), 2)
def test_create_with_parent_filediff_with_move_and_change(self):
"""Testing UploadDiffForm.create with a parent diff consisting of a
move/rename with content change
"""
revisions = [
b'93e6b3e8944c48737cb11a1e52b046fa30aea7a9',
b'4839fc480f47ca59cf05a9c39410ea744d1e17a2',
b'04861c126cfebd7e7cb93045ab0bff4a7acc4cf2',
]
parent_diff = SimpleUploadedFile(
'parent_diff',
(b'diff --git a/foo b/bar\n'
b'similarity index 55%%\n'
b'rename from foo\n'
b'rename to bar\n'
b'index %s..%s 100644\n'
b'--- a/foo\n'
b'+++ b/bar\n'
b'@@ -1,2 +1,3 @@\n'
b' Foo\n'
b'+Bar\n') % (revisions[0], revisions[1]),
content_type='text/x-patch')
diff = SimpleUploadedFile(
'diff',
(b'diff --git a/bar b/bar\n'
b'index %s..%s 100644\n'
b'--- a/bar\n'
b'+++ b/bar\n'
b'@@ -1,3 +1,4 @@\n'
b' Foo\n'
b' Bar\n'
b'+Baz\n') % (revisions[1], revisions[2]),
content_type='text/x-patch')
repository = self.create_repository(tool_name='Test')
self.spy_on(repository.get_file_exists,
call_fake=lambda *args, **kwargs: True)
# We will only be making one call to get_file and we can fake it out.
self.spy_on(repository.get_file,
call_fake=lambda *args, **kwargs: b'Foo\n')
self.spy_on(patch)
form = UploadDiffForm(
repository=repository,
data={
'basedir': '/',
},
files={
'path': diff,
'parent_diff_path': parent_diff,
})
self.assertTrue(form.is_valid())
diffset = form.create()
self.assertEqual(diffset.files.count(), 1)
f = diffset.files.get()
self.assertEqual(f.source_revision, revisions[0])
self.assertEqual(f.dest_detail, revisions[2])
original_file = get_original_file(f, None, ['ascii'])
self.assertEqual(original_file, b'Foo\nBar\n')
self.assertTrue(patch.spy.called)
patched_file = get_patched_file(original_file, f, None)
self.assertEqual(patched_file, b'Foo\nBar\nBaz\n')
self.assertEqual(len(patch.spy.calls), 2)
def get_chunks_uncached(self):
"""Yield the list of chunks, bypassing the cache."""
base_filediff = self.base_filediff
filediff = self.filediff
interfilediff = self.interfilediff
request = self.request
old = get_original_file(filediff=filediff, request=request)
new = get_patched_file(source_data=old,
filediff=filediff,
request=request)
old_encoding_list = get_filediff_encodings(filediff)
new_encoding_list = old_encoding_list
if base_filediff is not None:
# The diff is against a commit that:
#
# 1. Follows the first commit in a series (the first won't have
# a base_commit/base_filediff that can be looked up)
#
# 2. Follows a commit that modifies this file, or is the base
# commit that modifies this file.
#
# We'll be diffing against the patched version of this commit's
# version of the file.
old = get_original_file(filediff=base_filediff, request=request)
old = get_patched_file(source_data=old,
filediff=base_filediff,
request=request)
old_encoding_list = get_filediff_encodings(base_filediff)
elif filediff.commit_id:
# This diff is against a commit, but no previous FileDiff
# modifying this file could be found. As per the above comment,
# this could end up being the very first commit in a series, or
# it might not have been modified in the base commit or any
# previous commit.
#
# We'll need to fetch the first ancestor of this file in the
# commit history, if we can find one. We'll base the "old" version
# of the file on the original version of this commit, meaning that
# this commit and all modifications since will be shown as "new".
# Basically, viewing the upstream of the file, before any commits.
#
# This should be safe because, without a base_filediff, there
# should be no older commit containing modifications that we want
# to diff against. This would be the first one, and we're using
# its upstream changes.
ancestors = filediff.get_ancestors(minimal=True)
if ancestors:
ancestor_filediff = ancestors[0]
old = get_original_file(filediff=ancestor_filediff,
request=request)
old_encoding_list = get_filediff_encodings(ancestor_filediff)
# Check whether we have a SHA256 checksum first. They were introduced
# in Review Board 4.0, long after SHA1 checksums. If we already have
# a SHA256 checksum, then we'll also have a SHA1 checksum, but the
# inverse is not true.
if filediff.orig_sha256 is None:
if filediff.orig_sha1 is None:
filediff.extra_data.update({
'orig_sha1': self._get_sha1(old),
'patched_sha1': self._get_sha1(new),
})
filediff.extra_data.update({
'orig_sha256': self._get_sha256(old),
'patched_sha256': self._get_sha256(new),
})
filediff.save(update_fields=['extra_data'])
if interfilediff:
old = new
old_encoding_list = new_encoding_list
interdiff_orig = get_original_file(filediff=interfilediff,
request=request)
new = get_patched_file(source_data=interdiff_orig,
filediff=interfilediff,
request=request)
new_encoding_list = get_filediff_encodings(interfilediff)
# Check whether we have a SHA256 checksum first. They were
# introduced in Review Board 4.0, long after SHA1 checksums. If we
# already have a SHA256 checksum, then we'll also have a SHA1
# checksum, but the inverse is not true.
if interfilediff.orig_sha256 is None:
if interfilediff.orig_sha1 is None:
interfilediff.extra_data.update({
'orig_sha1':
self._get_sha1(interdiff_orig),
'patched_sha1':
self._get_sha1(new),
})
interfilediff.extra_data.update({
'orig_sha256':
self._get_sha256(interdiff_orig),
'patched_sha256':
self._get_sha256(new),
})
interfilediff.save(update_fields=['extra_data'])
elif self.force_interdiff:
# Basically, revert the change.
old, new = new, old
old_encoding_list, new_encoding_list = \
new_encoding_list, old_encoding_list
if interfilediff:
log_timer = log_timed(
"Generating diff chunks for interdiff ids %s-%s (%s)" %
(filediff.id, interfilediff.id, filediff.source_file),
request=request)
else:
log_timer = log_timed(
"Generating diff chunks for filediff id %s (%s)" %
(filediff.id, filediff.source_file),
request=request)
for chunk in self.generate_chunks(old=old,
new=new,
old_encoding_list=old_encoding_list,
new_encoding_list=new_encoding_list):
yield chunk
log_timer.done()
if (not interfilediff and not self.base_filediff
and not self.force_interdiff):
insert_count = self.counts['insert']
delete_count = self.counts['delete']
replace_count = self.counts['replace']
equal_count = self.counts['equal']
filediff.set_line_counts(
insert_count=insert_count,
delete_count=delete_count,
replace_count=replace_count,
equal_count=equal_count,
total_line_count=(insert_count + delete_count + replace_count +
equal_count))
def test_create_with_parent_filediff_with_new_file(self):
"""Testing UploadDiffForm.create with a parent diff consisting of a
newly-introduced file
"""
revisions = [
b'0000000000000000000000000000000000000000',
b'9b32edcd37a88c6ada91efc562afa637ccfdad36',
b'8a567d328293f85d68332bc693b0a98869b23b47',
]
parent_diff = SimpleUploadedFile('parent_diff',
(b'diff --git a/foo b/foo\n'
b'new file mode 100644\n'
b'index %s..%s\n'
b'--- /dev/null\n'
b'+++ b/foo\n'
b'@@ -0,0 +1,2 @@\n'
b'+Foo\n'
b'+Bar\n') %
(revisions[0], revisions[1]),
content_type='text/x-patch')
diff = SimpleUploadedFile('diff',
(b'diff --git a/foo b/foo\n'
b'index %s..%s 100644\n'
b'--- a/foo\n'
b'+++ b/foo\n'
b'@@ -1,3 +1,4 @@\n'
b' Foo\n'
b' Bar\n'
b'+Baz\n') % (revisions[1], revisions[2]),
content_type='text/x-patch')
repository = self.create_repository(tool_name='Test')
self.spy_on(repository.get_file_exists,
call_fake=lambda *args, **kwargs: True)
# We will only be making one call to get_file and we can fake it out.
self.spy_on(repository.get_file,
call_fake=lambda *args, **kwargs: b'Foo\n')
self.spy_on(patch)
form = UploadDiffForm(repository=repository,
data={
'basedir': '/',
},
files={
'parent_diff_path': parent_diff,
'path': diff,
})
self.assertTrue(form.is_valid())
diffset = form.create()
self.assertEqual(diffset.files.count(), 1)
filediff = diffset.files.get()
self.assertEqual(filediff.source_file, 'foo')
self.assertEqual(filediff.dest_file, 'foo')
self.assertEqual(filediff.source_revision,
revisions[1].decode('utf-8'))
self.assertEqual(filediff.dest_detail, revisions[2].decode('utf-8'))
self.assertEqual(
filediff.extra_data, {
'__parent_diff_empty': False,
'is_symlink': False,
'new_unix_mode': '100644',
'old_unix_mode': '100644',
'parent_source_filename': '/foo',
'parent_source_revision': 'PRE-CREATION',
'raw_delete_count': 0,
'raw_insert_count': 1,
})
# Double-check the types.
self.assertIsInstance(filediff.extra_data['parent_source_filename'],
str)
self.assertIsInstance(filediff.extra_data['parent_source_revision'],
str)
original_file = get_original_file(filediff=filediff,
request=None,
encoding_list=['ascii'])
self.assertEqual(original_file, b'Foo\nBar\n')
self.assertSpyCalled(patch)
patched_file = get_patched_file(source_data=original_file,
filediff=filediff)
self.assertEqual(patched_file, b'Foo\nBar\nBaz\n')
self.assertEqual(len(patch.calls), 2)
def test_create_with_parent_filediff_with_move_and_change(self):
"""Testing UploadDiffForm.create with a parent diff consisting of a
move/rename with content change
"""
revisions = [
b'5d36b88bb697a2d778f024048bafabd443d74503',
b'9b32edcd37a88c6ada91efc562afa637ccfdad36',
b'8a567d328293f85d68332bc693b0a98869b23b47',
]
parent_diff = SimpleUploadedFile('parent_diff',
(b'diff --git a/foo b/bar\n'
b'similarity index 55%%\n'
b'rename from foo\n'
b'rename to bar\n'
b'index %s..%s 100644\n'
b'--- a/foo\n'
b'+++ b/bar\n'
b'@@ -1,2 +1,3 @@\n'
b' Foo\n'
b'+Bar\n') %
(revisions[0], revisions[1]),
content_type='text/x-patch')
diff = SimpleUploadedFile('diff',
(b'diff --git a/bar b/bar\n'
b'index %s..%s 100644\n'
b'--- a/bar\n'
b'+++ b/bar\n'
b'@@ -1,3 +1,4 @@\n'
b' Foo\n'
b' Bar\n'
b'+Baz\n') % (revisions[1], revisions[2]),
content_type='text/x-patch')
repository = self.create_repository(tool_name='Test')
self.spy_on(repository.get_file_exists,
call_fake=lambda *args, **kwargs: True)
# We will only be making one call to get_file and we can fake it out.
self.spy_on(repository.get_file,
call_fake=lambda *args, **kwargs: b'Foo\n')
self.spy_on(patch)
form = UploadDiffForm(repository=repository,
data={
'basedir': '/',
},
files={
'path': diff,
'parent_diff_path': parent_diff,
})
self.assertTrue(form.is_valid())
diffset = form.create()
self.assertEqual(diffset.files.count(), 1)
filediff = diffset.files.get()
self.assertEqual(filediff.source_file, 'bar')
self.assertEqual(filediff.dest_file, 'bar')
self.assertEqual(filediff.source_revision,
revisions[1].decode('utf-8'))
self.assertEqual(filediff.dest_detail, revisions[2].decode('utf-8'))
self.assertEqual(
filediff.extra_data, {
'__parent_diff_empty': False,
'is_symlink': False,
'new_unix_mode': '100644',
'old_unix_mode': '100644',
'parent_moved': True,
'parent_source_filename': '/foo',
'parent_source_revision': revisions[0].decode('utf-8'),
'raw_delete_count': 0,
'raw_insert_count': 1,
})
original_file = get_original_file(filediff=filediff,
request=None,
encoding_list=['ascii'])
self.assertEqual(original_file, b'Foo\nBar\n')
self.assertTrue(patch.spy.called)
patched_file = get_patched_file(source_data=original_file,
filediff=filediff)
self.assertEqual(patched_file, b'Foo\nBar\nBaz\n')
self.assertEqual(len(patch.spy.calls), 2)
def get_chunks_uncached(self):
"""Yield the list of chunks, bypassing the cache."""
old = get_original_file(self.filediff, self.request,
self.encoding_list)
new = get_patched_file(old, self.filediff, self.request)
if self.base_filediff is not None:
# The diff is against a commit that:
#
# 1. Follows the first commit in a series (the first won't have
# a base_commit/base_filediff that can be looked up)
#
# 2. Follows a commit that modifies this file, or is the base
# commit that modifies this file.
#
# We'll be diffing against the patched version of this commit's
# version of the file.
old = get_original_file(self.base_filediff, self.request,
self.encoding_list)
old = get_patched_file(old, self.base_filediff, self.request)
elif self.filediff.commit_id:
# This diff is against a commit, but no previous FileDiff
# modifying this file could be found. As per the above comment,
# this could end up being the very first commit in a series, or
# it might not have been modified in the base commit or any
# previous commit.
#
# We'll need to fetch the first ancestor of this file in the
# commit history, if we can find one. We'll base the "old" version
# of the file on the original version of this commit, meaning that
# this commit and all modifications since will be shown as "new".
# Basically, viewing the upstream of the file, before any commits.
#
# This should be safe because, without a base_filediff, there
# should be no older commit containing modifications that we want
# to diff against. This would be the first one, and we're using
# its upstream changes.
ancestors = self.filediff.get_ancestors(minimal=True)
if ancestors:
old = get_original_file(ancestors[0],
self.request,
self.encoding_list)
if self.filediff.orig_sha1 is None:
self.filediff.extra_data.update({
'orig_sha1': self._get_checksum(old),
'patched_sha1': self._get_checksum(new),
})
self.filediff.save(update_fields=['extra_data'])
if self.interfilediff:
old = new
interdiff_orig = get_original_file(self.interfilediff,
self.request,
self.encoding_list)
new = get_patched_file(interdiff_orig, self.interfilediff,
self.request)
if self.interfilediff.orig_sha1 is None:
self.interfilediff.extra_data.update({
'orig_sha1': self._get_checksum(interdiff_orig),
'patched_sha1': self._get_checksum(new),
})
self.interfilediff.save(update_fields=['extra_data'])
elif self.force_interdiff:
# Basically, revert the change.
old, new = new, old
if self.interfilediff:
log_timer = log_timed(
"Generating diff chunks for interdiff ids %s-%s (%s)" %
(self.filediff.id, self.interfilediff.id,
self.filediff.source_file),
request=self.request)
else:
log_timer = log_timed(
"Generating diff chunks for self.filediff id %s (%s)" %
(self.filediff.id, self.filediff.source_file),
request=self.request)
for chunk in self.generate_chunks(old, new):
yield chunk
log_timer.done()
if (not self.interfilediff and
not self.base_filediff and
not self.force_interdiff):
insert_count = self.counts['insert']
delete_count = self.counts['delete']
replace_count = self.counts['replace']
equal_count = self.counts['equal']
self.filediff.set_line_counts(
insert_count=insert_count,
delete_count=delete_count,
replace_count=replace_count,
equal_count=equal_count,
total_line_count=(insert_count + delete_count +
replace_count + equal_count))
def _get_chunks_uncached(self):
"""Returns the list of chunks, bypassing the cache."""
old = get_original_file(self.filediff, self.request)
new = get_patched_file(old, self.filediff, self.request)
if self.interfilediff:
old = new
interdiff_orig = get_original_file(self.interfilediff,
self.request)
new = get_patched_file(interdiff_orig, self.interfilediff,
self.request)
elif self.force_interdiff:
# Basically, revert the change.
old, new = new, old
encoding = self.diffset.repository.encoding or 'iso-8859-15'
old = self._convert_to_utf8(old, encoding)
new = self._convert_to_utf8(new, encoding)
# Normalize the input so that if there isn't a trailing newline, we add
# it.
if old and old[-1] != '\n':
old += '\n'
if new and new[-1] != '\n':
new += '\n'
a = self.NEWLINES_RE.split(old or '')
b = self.NEWLINES_RE.split(new or '')
# Remove the trailing newline, now that we've split this. This will
# prevent a duplicate line number at the end of the diff.
del a[-1]
del b[-1]
a_num_lines = len(a)
b_num_lines = len(b)
markup_a = markup_b = None
if self._get_enable_syntax_highlighting(old, new, a, b):
repository = self.filediff.diffset.repository
tool = repository.get_scmtool()
source_file = \
tool.normalize_path_for_display(self.filediff.source_file)
dest_file = \
tool.normalize_path_for_display(self.filediff.dest_file)
try:
# TODO: Try to figure out the right lexer for these files
# once instead of twice.
markup_a = self._apply_pygments(old or '', source_file)
markup_b = self._apply_pygments(new or '', dest_file)
except:
pass
if not markup_a:
markup_a = self.NEWLINES_RE.split(escape(old))
if not markup_b:
markup_b = self.NEWLINES_RE.split(escape(new))
siteconfig = SiteConfiguration.objects.get_current()
ignore_space = True
for pattern in siteconfig.get('diffviewer_include_space_patterns'):
if fnmatch.fnmatch(self.filename, pattern):
ignore_space = False
break
self.differ = get_differ(a, b, ignore_space=ignore_space,
compat_version=self.diffset.diffcompat)
self.differ.add_interesting_lines_for_headers(self.filename)
context_num_lines = siteconfig.get("diffviewer_context_num_lines")
collapse_threshold = 2 * context_num_lines + 3
if self.interfilediff:
log_timer = log_timed(
"Generating diff chunks for interdiff ids %s-%s (%s)" %
(self.filediff.id, self.interfilediff.id,
self.filediff.source_file),
request=self.request)
else:
log_timer = log_timed(
"Generating diff chunks for self.filediff id %s (%s)" %
(self.filediff.id, self.filediff.source_file),
request=self.request)
line_num = 1
opcodes_generator = get_diff_opcode_generator(self.differ,
self.filediff,
self.interfilediff)
for tag, i1, i2, j1, j2, meta in opcodes_generator:
old_lines = markup_a[i1:i2]
new_lines = markup_b[j1:j2]
num_lines = max(len(old_lines), len(new_lines))
self._cur_meta = meta
lines = map(self._diff_line,
range(line_num, line_num + num_lines),
range(i1 + 1, i2 + 1), range(j1 + 1, j2 + 1),
a[i1:i2], b[j1:j2], old_lines, new_lines)
self._cur_meta = None
if tag == 'equal' and num_lines > collapse_threshold:
last_range_start = num_lines - context_num_lines
if line_num == 1:
yield self._new_chunk(lines, 0, last_range_start, True)
yield self._new_chunk(lines, last_range_start, num_lines)
else:
yield self._new_chunk(lines, 0, context_num_lines)
if i2 == a_num_lines and j2 == b_num_lines:
yield self._new_chunk(lines, context_num_lines,
num_lines, True)
else:
yield self._new_chunk(lines, context_num_lines,
last_range_start, True)
yield self._new_chunk(lines, last_range_start,
num_lines)
else:
yield self._new_chunk(lines, 0, num_lines, False, tag, meta)
line_num += num_lines
log_timer.done()
def test_create_with_parent_filediff_with_move_and_no_change(self):
"""Testing UploadDiffForm.create with a parent diff consisting only
of a move/rename without content change
"""
revisions = [
b'93e6b3e8944c48737cb11a1e52b046fa30aea7a9',
b'4839fc480f47ca59cf05a9c39410ea744d1e17a2',
]
parent_diff = SimpleUploadedFile('parent_diff',
(b'diff --git a/foo b/bar\n'
b'similarity index 100%%\n'
b'rename from foo\n'
b'rename to bar\n'),
content_type='text/x-patch')
diff = SimpleUploadedFile('diff',
(b'diff --git a/bar b/bar\n'
b'index %s..%s 100644\n'
b'--- a/bar\n'
b'+++ b/bar\n'
b'@@ -1,2 +1,3 @@\n'
b' Foo\n'
b'+Bar\n') % (revisions[0], revisions[1]),
content_type='text/x-patch')
repository = self.create_repository(tool_name='Test')
self.spy_on(repository.get_file_exists,
call_fake=lambda *args, **kwargs: True)
# We will only be making one call to get_file and we can fake it out.
self.spy_on(repository.get_file,
call_fake=lambda *args, **kwargs: b'Foo\n')
self.spy_on(patch)
form = UploadDiffForm(repository=repository,
data={
'basedir': '/',
},
files={
'path': diff,
'parent_diff_path': parent_diff,
})
self.assertTrue(form.is_valid())
diffset = form.create()
self.assertEqual(diffset.files.count(), 1)
f = diffset.files.get()
self.assertEqual(f.source_revision, revisions[0].decode('utf-8'))
self.assertEqual(f.dest_detail, revisions[1].decode('utf-8'))
# We shouldn't call out to patch because the parent diff is just a
# rename.
original_file = get_original_file(filediff=f,
request=None,
encoding_list=['ascii'])
self.assertEqual(original_file, b'Foo\n')
self.assertFalse(patch.spy.called)
patched_file = get_patched_file(source_data=original_file, filediff=f)
self.assertEqual(patched_file, b'Foo\nBar\n')
self.assertTrue(patch.spy.called)
