def get_chunks_uncached(self): """Yield the list of chunks, bypassing the cache.""" old = get_original_file(self.filediff, self.request, self.encoding_list) new = get_patched_file(old, self.filediff, self.request) if self.filediff.orig_sha1 is None: self.filediff.extra_data.update( {"orig_sha1": self._get_checksum(old), "patched_sha1": self._get_checksum(new)} ) self.filediff.save(update_fields=["extra_data"]) if self.interfilediff: old = new interdiff_orig = get_original_file(self.interfilediff, self.request, self.encoding_list) new = get_patched_file(interdiff_orig, self.interfilediff, self.request) if self.interfilediff.orig_sha1 is None: self.interfilediff.extra_data.update( {"orig_sha1": self._get_checksum(interdiff_orig), "patched_sha1": self._get_checksum(new)} ) self.interfilediff.save(update_fields=["extra_data"]) elif self.force_interdiff: # Basically, revert the change. old, new = new, old if self.interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (self.filediff.id, self.interfilediff.id, self.filediff.source_file), request=self.request, ) else: log_timer = log_timed( "Generating diff chunks for self.filediff id %s (%s)" % (self.filediff.id, self.filediff.source_file), request=self.request, ) for chunk in self.generate_chunks(old, new): yield chunk log_timer.done() if not self.interfilediff and not self.force_interdiff: insert_count = self.counts["insert"] delete_count = self.counts["delete"] replace_count = self.counts["replace"] equal_count = self.counts["equal"] self.filediff.set_line_counts( insert_count=insert_count, delete_count=delete_count, replace_count=replace_count, equal_count=equal_count, total_line_count=(insert_count + delete_count + replace_count + equal_count), )
def fetch_file(file, revision): log_timer = log_timed("Fetching file '%s' r%s from %s" % (file, revision, repository)) data = tool.get_file(file, revision) data = convert_line_endings(data) log_timer.done() return data
def _get_file_uncached(self, path, revision, request): """Internal function for fetching an uncached file. This is called by get_file if the file isn't already in the cache. """ fetching_file.send(sender=self, path=path, revision=revision, request=request) log_timer = log_timed("Fetching file '%s' r%s from %s" % (path, revision, self), request=request) hosting_service = self.hosting_service if hosting_service: data = hosting_service.get_file(self, path, revision) else: data = self.get_scmtool().get_file(path, revision) log_timer.done() fetched_file.send(sender=self, path=path, revision=revision, request=request, data=data) return data
def patch(diff, file, filename, request=None): """Apply a diff to a file. Delegates out to `patch` because noone except Larry Wall knows how to patch.""" log_timer = log_timed("Patching file %s" % filename, request=request) if diff.strip() == "": # Someone uploaded an unchanged file. Return the one we're patching. return file # Prepare the temporary directory if none is available tempdir = tempfile.mkdtemp(prefix='reviewboard.') (fd, oldfile) = tempfile.mkstemp(dir=tempdir) f = os.fdopen(fd, "w+b") f.write(convert_line_endings(file)) f.close() diff = convert_line_endings(diff) newfile = '%s-new' % oldfile process = subprocess.Popen(['patch', '-o', newfile, oldfile], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) with controlled_subprocess("patch", process) as p: p.stdin.write(diff) p.stdin.close() patch_output = p.stdout.read() failure = p.wait() if failure: f = open("%s.diff" % (os.path.join(tempdir, os.path.basename(filename))), "w") f.write(diff) f.close() log_timer.done() # FIXME: This doesn't provide any useful error report on why the patch # failed to apply, which makes it hard to debug. We might also want to # have it clean up if DEBUG=False raise Exception(_("The patch to '%s' didn't apply cleanly. The temporary " + "files have been left in '%s' for debugging purposes.\n" + "`patch` returned: %s") % (filename, tempdir, patch_output)) f = open(newfile, "r") data = f.read() f.close() os.unlink(oldfile) os.unlink(newfile) os.rmdir(tempdir) log_timer.done() return data
def process_request(self, request): """ Processes an incoming request. This will set up logging. """ if getattr(settings, 'LOGGING_PAGE_TIMES', False): request._page_timedloginfo = \ log_timed('Page request: HTTP %s %s (by %s)' % (request.method, request.path, request.user)) if ('profiling' in request.GET and getattr(settings, "LOGGING_ALLOW_PROFILING", False)): settings.DEBUG = True
def _get_file_uncached(self, path, revision, base_commit_id, request): """Internal function for fetching an uncached file. This is called by get_file if the file isn't already in the cache. """ fetching_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request) if base_commit_id: timer_msg = "Fetching file '%s' r%s (base commit ID %s) from %s" \ % (path, revision, base_commit_id, self) else: timer_msg = "Fetching file '%s' r%s from %s" \ % (path, revision, self) log_timer = log_timed(timer_msg, request=request) hosting_service = self.hosting_service if hosting_service: data = hosting_service.get_file( self, path, revision, base_commit_id=base_commit_id) else: tool = self.get_scmtool() argspec = inspect.getargspec(tool.get_file) if argspec.keywords is None: warnings.warn('SCMTool.get_file() must take keyword ' 'arguments, signature for %s is deprecated.' % tool.name, RemovedInReviewBoard40Warning) data = tool.get_file(path, revision) else: data = tool.get_file(path, revision, base_commit_id=base_commit_id) log_timer.done() fetched_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request, data=data) return data
def apply_pygments(data, filename): # Sometimes pygments spins forever trying to highlight something, give # it a couple seconds then give up. def timeout_handler(signum, frame): raise TimeoutException() result = None signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(2) try: # XXX Guessing is preferable but really slow, especially on XML # files. log_timer = log_timed("Lexer guessing for '%s'" % filename) if filename.endswith(".xml"): lexer = get_lexer_for_filename(filename, stripnl=False, encoding='utf-8') else: lexer = guess_lexer_for_filename(filename, data, stripnl=False) log_timer.done() try: # This is only available in 0.7 and higher lexer.add_filter('codetagify') except AttributeError: pass log_timer = log_timed("Syntax highlighting file '%s' with lexer %s" % (filename, lexer.name)) result = pygments.highlight(data, lexer, NoWrapperHtmlFormatter()).splitlines() log_timer.done() signal.alarm(0) except TimeoutException: logging.warn("Timed out trying to highlight data for file '%s' with lexer %s" % (filename, lexer.name)) return result
def _get_file_uncached(self, path, revision, base_commit_id, request): """Internal function for fetching an uncached file. This is called by get_file if the file isn't already in the cache. """ fetching_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request) if base_commit_id: timer_msg = "Fetching file '%s' r%s (base commit ID %s) from %s" \ % (path, revision, base_commit_id, self) else: timer_msg = "Fetching file '%s' r%s from %s" \ % (path, revision, self) log_timer = log_timed(timer_msg, request=request) hosting_service = self.hosting_service if hosting_service: data = hosting_service.get_file( self, path, revision, base_commit_id=base_commit_id) else: try: data = self.get_scmtool().get_file(path, revision) except FileNotFoundError: if base_commit_id: # Some funky workflows with mq (mercurial) can cause issues # with parent diffs. If we didn't find it with the parsed # revision, and there's a base commit ID, try that. data = self.get_scmtool().get_file(path, base_commit_id) else: raise log_timer.done() fetched_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request, data=data) return data
def _get_file_uncached(self, path, revision, base_commit_id, request): """Internal function for fetching an uncached file. This is called by get_file if the file isn't already in the cache. """ fetching_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request) if base_commit_id: timer_msg = "Fetching file '%s' r%s (base commit ID %s) from %s" \ % (path, revision, base_commit_id, self) else: timer_msg = "Fetching file '%s' r%s from %s" \ % (path, revision, self) log_timer = log_timed(timer_msg, request=request) hosting_service = self.hosting_service if hosting_service: data = hosting_service.get_file( self, path, revision, base_commit_id=base_commit_id) else: data = self.get_scmtool().get_file(path, revision) log_timer.done() fetched_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request, data=data) return data
def _get_chunks_uncached(self): """Returns the list of chunks, bypassing the cache.""" encoding_list = self.diffset.repository.get_encoding_list() old = get_original_file(self.filediff, self.request, encoding_list) new = get_patched_file(old, self.filediff, self.request) if self.interfilediff: old = new interdiff_orig = get_original_file(self.interfilediff, self.request, encoding_list) new = get_patched_file(interdiff_orig, self.interfilediff, self.request) elif self.force_interdiff: # Basically, revert the change. old, new = new, old old = convert_to_unicode(old, encoding_list)[1] new = convert_to_unicode(new, encoding_list)[1] # Normalize the input so that if there isn't a trailing newline, we add # it. if old and old[-1] != '\n': old += '\n' if new and new[-1] != '\n': new += '\n' a = self.NEWLINES_RE.split(old or '') b = self.NEWLINES_RE.split(new or '') # Remove the trailing newline, now that we've split this. This will # prevent a duplicate line number at the end of the diff. del a[-1] del b[-1] a_num_lines = len(a) b_num_lines = len(b) markup_a = markup_b = None if self._get_enable_syntax_highlighting(old, new, a, b): repository = self.filediff.diffset.repository tool = repository.get_scmtool() source_file = \ tool.normalize_path_for_display(self.filediff.source_file) dest_file = \ tool.normalize_path_for_display(self.filediff.dest_file) try: # TODO: Try to figure out the right lexer for these files # once instead of twice. markup_a = self._apply_pygments(old or '', source_file) markup_b = self._apply_pygments(new or '', dest_file) except: pass if not markup_a: markup_a = self.NEWLINES_RE.split(escape(old)) if not markup_b: markup_b = self.NEWLINES_RE.split(escape(new)) siteconfig = SiteConfiguration.objects.get_current() ignore_space = True for pattern in siteconfig.get('diffviewer_include_space_patterns'): if fnmatch.fnmatch(self.filename, pattern): ignore_space = False break self.differ = get_differ(a, b, ignore_space=ignore_space, compat_version=self.diffset.diffcompat) self.differ.add_interesting_lines_for_headers(self.filename) context_num_lines = siteconfig.get("diffviewer_context_num_lines") collapse_threshold = 2 * context_num_lines + 3 if self.interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (self.filediff.id, self.interfilediff.id, self.filediff.source_file), request=self.request) else: log_timer = log_timed( "Generating diff chunks for self.filediff id %s (%s)" % (self.filediff.id, self.filediff.source_file), request=self.request) line_num = 1 opcodes_generator = get_diff_opcode_generator(self.differ, self.filediff, self.interfilediff) counts = { 'equal': 0, 'replace': 0, 'insert': 0, 'delete': 0, } for tag, i1, i2, j1, j2, meta in opcodes_generator: old_lines = markup_a[i1:i2] new_lines = markup_b[j1:j2] num_lines = max(len(old_lines), len(new_lines)) lines = map(functools.partial(self._diff_line, tag, meta), range(line_num, line_num + num_lines), range(i1 + 1, i2 + 1), range(j1 + 1, j2 + 1), a[i1:i2], b[j1:j2], old_lines, new_lines) counts[tag] += num_lines if tag == 'equal' and num_lines > collapse_threshold: last_range_start = num_lines - context_num_lines if line_num == 1: yield self._new_chunk(lines, 0, last_range_start, True) yield self._new_chunk(lines, last_range_start, num_lines) else: yield self._new_chunk(lines, 0, context_num_lines) if i2 == a_num_lines and j2 == b_num_lines: yield self._new_chunk(lines, context_num_lines, num_lines, True) else: yield self._new_chunk(lines, context_num_lines, last_range_start, True) yield self._new_chunk(lines, last_range_start, num_lines) else: yield self._new_chunk(lines, 0, num_lines, False, tag, meta) line_num += num_lines log_timer.done() if not self.interfilediff: insert_count = counts['insert'] delete_count = counts['delete'] replace_count = counts['replace'] equal_count = counts['equal'] self.filediff.set_line_counts( insert_count=insert_count, delete_count=delete_count, replace_count=replace_count, equal_count=equal_count, total_line_count=(insert_count + delete_count + replace_count + equal_count))
def get_diff_files(diffset, filediff=None, interdiffset=None): """Generates a list of files that will be displayed in a diff. This will go through the given diffset/interdiffset, or a given filediff within that diffset, and generate the list of files that will be displayed. This file list will contain a bunch of metadata on the files, such as the index, original/modified names, revisions, associated filediffs/diffsets, and so on. This can be used along with populate_diff_chunks to build a full list containing all diff chunks used for rendering a side-by-side diff. """ if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id)) else: log_timer = log_timed("Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id)) else: filediffs = diffset.files.select_related().all() if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id)) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id) # A map used to quickly look up the equivalent interfilediff given a # source file. interdiff_map = {} if interdiffset: for interfilediff in interdiffset.files.all(): if not filediff or \ filediff.source_file == interfilediff.source_file: interdiff_map[interfilediff.source_file] = interfilediff # In order to support interdiffs properly, we need to display diffs # on every file in the union of both diffsets. Iterating over one diffset # or the other doesn't suffice. # # We build a list of parts containing the source filediff, the interdiff # filediff (if specified), and whether to force showing an interdiff # (in the case where a file existed in the source filediff but was # reverted in the interdiff). filediff_parts = [] for filediff in filediffs: interfilediff = None if filediff.source_file in interdiff_map: interfilediff = interdiff_map[filediff.source_file] del(interdiff_map[filediff.source_file]) filediff_parts.append((filediff, interfilediff, interdiffset != None)) if interdiffset: # We've removed everything in the map that we've already found. # What's left are interdiff files that are new. They have no file # to diff against. # # The end result is going to be a view that's the same as when you're # viewing a standard diff. As such, we can pretend the interdiff is # the source filediff and not specify an interdiff. Keeps things # simple, code-wise, since we really have no need to special-case # this. filediff_parts += [(interdiff, None, False) for interdiff in interdiff_map.values()] files = [] for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts newfile = (filediff.source_revision == PRE_CREATION) if interdiffset: # First, find out if we want to even process this one. # We only process if there's a difference in files. if (filediff and interfilediff and filediff.diff == interfilediff.diff): continue source_revision = "Diff Revision %s" % diffset.revision if not interfilediff and force_interdiff: dest_revision = "Diff Revision %s - File Reverted" % \ interdiffset.revision else: dest_revision = "Diff Revision %s" % interdiffset.revision else: source_revision = get_revision_str(filediff.source_revision) if newfile: dest_revision = NEW_FILE_STR else: dest_revision = NEW_CHANGE_STR i = filediff.source_file.rfind('/') if i != -1: basepath = filediff.source_file[:i] basename = filediff.source_file[i + 1:] else: basepath = "" basename = filediff.source_file tool = filediff.diffset.repository.get_scmtool() depot_filename = tool.normalize_path_for_display(filediff.source_file) dest_filename = tool.normalize_path_for_display(filediff.dest_file) files.append({ 'depot_filename': depot_filename, 'dest_filename': dest_filename or depot_filename, 'basename': basename, 'basepath': basepath, 'revision': source_revision, 'dest_revision': dest_revision, 'filediff': filediff, 'interfilediff': interfilediff, 'force_interdiff': force_interdiff, 'binary': filediff.binary, 'deleted': filediff.deleted, 'moved': filediff.moved, 'newfile': newfile, 'index': len(files), 'chunks_loaded': False, }) def cmp_file(x, y): # Sort based on basepath in asc order if x["basepath"] != y["basepath"]: return cmp(x["basepath"], y["basepath"]) # Sort based on filename in asc order, then based on extension in desc # order, to make *.h be ahead of *.c/cpp x_file, x_ext = os.path.splitext(x["basename"]) y_file, y_ext = os.path.splitext(y["basename"]) if x_file != y_file: return cmp(x_file, y_file) else: return cmp(y_ext, x_ext) files.sort(cmp_file) log_timer.done() return files
def patch(diff, file, filename, request=None): """Apply a diff to a file. Delegates out to `patch` because noone except Larry Wall knows how to patch.""" log_timer = log_timed("Patching file %s" % filename, request=request) if diff.strip() == "": # Someone uploaded an unchanged file. Return the one we're patching. return file # Prepare the temporary directory if none is available tempdir = tempfile.mkdtemp(prefix='reviewboard.') (fd, oldfile) = tempfile.mkstemp(dir=tempdir) f = os.fdopen(fd, "w+b") f.write(convert_line_endings(file)) f.close() diff = convert_line_endings(diff) newfile = '%s-new' % oldfile process = subprocess.Popen(['patch', '-o', newfile, oldfile], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=tempdir) with controlled_subprocess("patch", process) as p: p.stdin.write(diff) p.stdin.close() patch_output = p.stdout.read() failure = p.wait() if failure: f = open( "%s.diff" % (os.path.join(tempdir, os.path.basename(filename))), "w") f.write(diff) f.close() log_timer.done() # FIXME: This doesn't provide any useful error report on why the patch # failed to apply, which makes it hard to debug. We might also want to # have it clean up if DEBUG=False raise Exception( _("The patch to '%(filename)s' didn't apply cleanly. The " "temporary files have been left in '%(tempdir)s' for debugging " "purposes.\n" "`patch` returned: %(output)s") % { 'filename': filename, 'tempdir': tempdir, 'output': patch_output, }) f = open(newfile, "r") data = f.read() f.close() os.unlink(oldfile) os.unlink(newfile) os.rmdir(tempdir) log_timer.done() return data
def get_chunks_uncached(self): """Yield the list of chunks, bypassing the cache.""" base_filediff = self.base_filediff filediff = self.filediff interfilediff = self.interfilediff request = self.request old = get_original_file(filediff=filediff, request=request) new = get_patched_file(source_data=old, filediff=filediff, request=request) old_encoding_list = get_filediff_encodings(filediff) new_encoding_list = old_encoding_list if base_filediff is not None: # The diff is against a commit that: # # 1. Follows the first commit in a series (the first won't have # a base_commit/base_filediff that can be looked up) # # 2. Follows a commit that modifies this file, or is the base # commit that modifies this file. # # We'll be diffing against the patched version of this commit's # version of the file. old = get_original_file(filediff=base_filediff, request=request) old = get_patched_file(source_data=old, filediff=base_filediff, request=request) old_encoding_list = get_filediff_encodings(base_filediff) elif filediff.commit_id: # This diff is against a commit, but no previous FileDiff # modifying this file could be found. As per the above comment, # this could end up being the very first commit in a series, or # it might not have been modified in the base commit or any # previous commit. # # We'll need to fetch the first ancestor of this file in the # commit history, if we can find one. We'll base the "old" version # of the file on the original version of this commit, meaning that # this commit and all modifications since will be shown as "new". # Basically, viewing the upstream of the file, before any commits. # # This should be safe because, without a base_filediff, there # should be no older commit containing modifications that we want # to diff against. This would be the first one, and we're using # its upstream changes. ancestors = filediff.get_ancestors(minimal=True) if ancestors: ancestor_filediff = ancestors[0] old = get_original_file(filediff=ancestor_filediff, request=request) old_encoding_list = get_filediff_encodings(ancestor_filediff) # Check whether we have a SHA256 checksum first. They were introduced # in Review Board 4.0, long after SHA1 checksums. If we already have # a SHA256 checksum, then we'll also have a SHA1 checksum, but the # inverse is not true. if filediff.orig_sha256 is None: if filediff.orig_sha1 is None: filediff.extra_data.update({ 'orig_sha1': self._get_sha1(old), 'patched_sha1': self._get_sha1(new), }) filediff.extra_data.update({ 'orig_sha256': self._get_sha256(old), 'patched_sha256': self._get_sha256(new), }) filediff.save(update_fields=['extra_data']) if interfilediff: old = new old_encoding_list = new_encoding_list interdiff_orig = get_original_file(filediff=interfilediff, request=request) new = get_patched_file(source_data=interdiff_orig, filediff=interfilediff, request=request) new_encoding_list = get_filediff_encodings(interfilediff) # Check whether we have a SHA256 checksum first. They were # introduced in Review Board 4.0, long after SHA1 checksums. If we # already have a SHA256 checksum, then we'll also have a SHA1 # checksum, but the inverse is not true. if interfilediff.orig_sha256 is None: if interfilediff.orig_sha1 is None: interfilediff.extra_data.update({ 'orig_sha1': self._get_sha1(interdiff_orig), 'patched_sha1': self._get_sha1(new), }) interfilediff.extra_data.update({ 'orig_sha256': self._get_sha256(interdiff_orig), 'patched_sha256': self._get_sha256(new), }) interfilediff.save(update_fields=['extra_data']) elif self.force_interdiff: # Basically, revert the change. old, new = new, old old_encoding_list, new_encoding_list = \ new_encoding_list, old_encoding_list if interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (filediff.id, interfilediff.id, filediff.source_file), request=request) else: log_timer = log_timed( "Generating diff chunks for filediff id %s (%s)" % (filediff.id, filediff.source_file), request=request) for chunk in self.generate_chunks(old=old, new=new, old_encoding_list=old_encoding_list, new_encoding_list=new_encoding_list): yield chunk log_timer.done() if (not interfilediff and not self.base_filediff and not self.force_interdiff): insert_count = self.counts['insert'] delete_count = self.counts['delete'] replace_count = self.counts['replace'] equal_count = self.counts['equal'] filediff.set_line_counts( insert_count=insert_count, delete_count=delete_count, replace_count=replace_count, equal_count=equal_count, total_line_count=(insert_count + delete_count + replace_count + equal_count))
def _get_file_uncached(self, path, revision, context): """Return a file from the repository, bypassing cache. This is called internally by :py:meth:`get_file` if the file isn't already in the cache. This will send the :py:data:`~reviewboard.scmtools.signals.fetching_file` signal before beginning a file fetch from the repository, and the :py:data:`~reviewboard.scmtools.signals.fetched_file` signal after. Args: path (unicode): The path to the file in the repository. revision (unicode): The revision of the file to retrieve. context (reviewboard.scmtools.core.FileLookupContext): Extra context used to help look up this file. Version Added: 4.0.5 Returns: bytes: The resulting file contents. """ request = context.request base_commit_id = context.base_commit_id fetching_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request, context=context) if base_commit_id: timer_msg = "Fetching file '%s' r%s (base commit ID %s) from %s" \ % (path, revision, base_commit_id, self) else: timer_msg = "Fetching file '%s' r%s from %s" \ % (path, revision, self) log_timer = log_timed(timer_msg, request=request) hosting_service = self.hosting_service if hosting_service: data = hosting_service.get_file(self, path, revision, base_commit_id=base_commit_id, context=context) assert isinstance( data, bytes), ('%s.get_file() must return a byte string, not %s' % (type(hosting_service).__name__, type(data))) else: tool = self.get_scmtool() data = tool.get_file(path, revision, base_commit_id=base_commit_id, context=context) assert isinstance( data, bytes), ('%s.get_file() must return a byte string, not %s' % (type(tool).__name__, type(data))) log_timer.done() fetched_file.send(sender=self, path=path, revision=revision, base_commit_id=base_commit_id, request=request, context=context, data=data) return data
def get_chunks(diffset, filediff, interfilediff, force_interdiff, enable_syntax_highlighting): def diff_line(vlinenum, oldlinenum, newlinenum, oldline, newline, oldmarkup, newmarkup): # This function accesses the variable meta, defined in an outer context. if (oldline and newline and len(oldline) <= STYLED_MAX_LINE_LEN and len(newline) <= STYLED_MAX_LINE_LEN and oldline != newline): oldregion, newregion = get_line_changed_regions(oldline, newline) else: oldregion = newregion = [] result = [vlinenum, oldlinenum or '', mark_safe(oldmarkup or ''), oldregion, newlinenum or '', mark_safe(newmarkup or ''), newregion, (oldlinenum, newlinenum) in meta['whitespace_lines']] if oldlinenum and oldlinenum in meta.get('moved', {}): destination = meta["moved"][oldlinenum] result.append(destination) elif newlinenum and newlinenum in meta.get('moved', {}): destination = meta["moved"][newlinenum] result.append(destination) return result def new_chunk(chunk_index, all_lines, start, end, collapsable=False, tag='equal', meta=None): if not meta: meta = {} left_headers = list(get_interesting_headers(differ, all_lines, start, end - 1, False)) right_headers = list(get_interesting_headers(differ, all_lines, start, end - 1, True)) meta['left_headers'] = left_headers meta['right_headers'] = right_headers lines = all_lines[start:end] numlines = len(lines) compute_chunk_last_header(lines, numlines, meta, last_header) if (collapsable and end < len(all_lines) and (last_header[0] or last_header[1])): meta['headers'] = list(last_header) return { 'index': chunk_index, 'lines': lines, 'numlines': numlines, 'change': tag, 'collapsable': collapsable, 'meta': meta, } def get_interesting_headers(differ, lines, start, end, is_modified_file): """Returns all headers for a region of a diff. This scans for all headers that fall within the specified range of the specified lines on both the original and modified files. """ possible_functions = differ.get_interesting_lines('header', is_modified_file) if not possible_functions: raise StopIteration try: if is_modified_file: last_index = last_header_index[1] i1 = lines[start][4] i2 = lines[end - 1][4] else: last_index = last_header_index[0] i1 = lines[start][1] i2 = lines[end - 1][1] except IndexError: raise StopIteration for i in xrange(last_index, len(possible_functions)): linenum, line = possible_functions[i] linenum += 1 if linenum > i2: break elif linenum >= i1: last_index = i yield (linenum, line) if is_modified_file: last_header_index[1] = last_index else: last_header_index[0] = last_index def apply_pygments(data, filename): # XXX Guessing is preferable but really slow, especially on XML # files. #if filename.endswith(".xml"): lexer = get_lexer_for_filename(filename, stripnl=False, encoding='utf-8') #else: # lexer = guess_lexer_for_filename(filename, data, stripnl=False) try: # This is only available in 0.7 and higher lexer.add_filter('codetagify') except AttributeError: pass return pygments.highlight(data, lexer, NoWrapperHtmlFormatter()).splitlines() # There are three ways this function is called: # # 1) filediff, no interfilediff # - Returns chunks for a single filediff. This is the usual way # people look at diffs in the diff viewer. # # In this mode, we get the original file based on the filediff # and then patch it to get the resulting file. # # This is also used for interdiffs where the source revision # has no equivalent modified file but the interdiff revision # does. It's no different than a standard diff. # # 2) filediff, interfilediff # - Returns chunks showing the changes between a source filediff # and the interdiff. # # This is the typical mode used when showing the changes # between two diffs. It requires that the file is included in # both revisions of a diffset. # # 3) filediff, no interfilediff, force_interdiff # - Returns chunks showing the changes between a source # diff and an unmodified version of the diff. # # This is used when the source revision in the diffset contains # modifications to a file which have then been reverted in the # interdiff revision. We don't actually have an interfilediff # in this case, so we have to indicate that we are indeed in # interdiff mode so that we can special-case this and not # grab a patched file for the interdiff version. assert filediff file = filediff.source_file old = get_original_file(filediff) new = get_patched_file(old, filediff) if interfilediff: old = new interdiff_orig = get_original_file(interfilediff) new = get_patched_file(interdiff_orig, interfilediff) elif force_interdiff: # Basically, revert the change. old, new = new, old encoding = diffset.repository.encoding or 'iso-8859-15' old = convert_to_utf8(old, encoding) new = convert_to_utf8(new, encoding) # Normalize the input so that if there isn't a trailing newline, we add # it. if old and old[-1] != '\n': old += '\n' if new and new[-1] != '\n': new += '\n' a = NEWLINES_RE.split(old or '') b = NEWLINES_RE.split(new or '') # Remove the trailing newline, now that we've split this. This will # prevent a duplicate line number at the end of the diff. del(a[-1]) del(b[-1]) a_num_lines = len(a) b_num_lines = len(b) markup_a = markup_b = None siteconfig = SiteConfiguration.objects.get_current() threshold = siteconfig.get('diffviewer_syntax_highlighting_threshold') if threshold and (a_num_lines > threshold or b_num_lines > threshold): enable_syntax_highlighting = False if enable_syntax_highlighting: # Very long files, especially XML files, can take a long time to # highlight. For files over a certain size, don't highlight them. if (len(old) > STYLED_MAX_LIMIT_BYTES or len(new) > STYLED_MAX_LIMIT_BYTES): enable_syntax_highlighting = False if enable_syntax_highlighting: # Don't style the file if we have any *really* long lines. # It's likely a minified file or data or something that doesn't # need styling, and it will just grind Review Board to a halt. for lines in (a, b): for line in lines: if len(line) > STYLED_MAX_LINE_LEN: enable_syntax_highlighting = False break if not enable_syntax_highlighting: break if enable_syntax_highlighting: repository = filediff.diffset.repository tool = repository.get_scmtool() source_file = tool.normalize_path_for_display(filediff.source_file) dest_file = tool.normalize_path_for_display(filediff.dest_file) try: # TODO: Try to figure out the right lexer for these files # once instead of twice. markup_a = apply_pygments(old or '', source_file) markup_b = apply_pygments(new or '', dest_file) except: pass if not markup_a: markup_a = NEWLINES_RE.split(escape(old)) if not markup_b: markup_b = NEWLINES_RE.split(escape(new)) linenum = 1 last_header = [None, None] last_header_index = [0, 0] ignore_space = True for pattern in siteconfig.get("diffviewer_include_space_patterns"): if fnmatch.fnmatch(file, pattern): ignore_space = False break differ = Differ(a, b, ignore_space=ignore_space, compat_version=diffset.diffcompat) # Register any regexes for interesting lines we may want to show. register_interesting_lines_for_filename(differ, file) # TODO: Make this back into a preference if people really want it. context_num_lines = siteconfig.get("diffviewer_context_num_lines") collapse_threshold = 2 * context_num_lines + 3 if interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (filediff.id, interfilediff.id, filediff.source_file)) else: log_timer = log_timed( "Generating diff chunks for filediff id %s (%s)" % (filediff.id, filediff.source_file)) chunk_index = 0 for tag, i1, i2, j1, j2, meta in opcodes_with_metadata(differ): oldlines = markup_a[i1:i2] newlines = markup_b[j1:j2] numlines = max(len(oldlines), len(newlines)) lines = map(diff_line, xrange(linenum, linenum + numlines), xrange(i1 + 1, i2 + 1), xrange(j1 + 1, j2 + 1), a[i1:i2], b[j1:j2], oldlines, newlines) if tag == 'equal' and numlines > collapse_threshold: last_range_start = numlines - context_num_lines if linenum == 1: yield new_chunk(chunk_index, lines, 0, last_range_start, True) chunk_index += 1 yield new_chunk(chunk_index, lines, last_range_start, numlines) chunk_index += 1 else: yield new_chunk(chunk_index, lines, 0, context_num_lines) chunk_index += 1 if i2 == a_num_lines and j2 == b_num_lines: yield new_chunk(chunk_index, lines, context_num_lines, numlines, True) chunk_index += 1 else: yield new_chunk(chunk_index, lines, context_num_lines, last_range_start, True) chunk_index += 1 yield new_chunk(chunk_index, lines, last_range_start, numlines) chunk_index += 1 else: yield new_chunk(chunk_index, lines, 0, numlines, False, tag, meta) chunk_index += 1 linenum += numlines log_timer.done()
def get_diff_files(diffset, filediff=None, interdiffset=None, enable_syntax_highlighting=True, load_chunks=True): if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed( "Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id) ) else: log_timer = log_timed( "Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id) ) else: filediffs = diffset.files.all() if interdiffset: log_timer = log_timed( "Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id) ) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id) # A map used to quickly look up the equivalent interfilediff given a # source file. interdiff_map = {} if interdiffset: for interfilediff in interdiffset.files.all(): if not filediff or filediff.source_file == interfilediff.source_file: interdiff_map[interfilediff.source_file] = interfilediff key_prefix = "diff-sidebyside-" if enable_syntax_highlighting: key_prefix += "hl-" # In order to support interdiffs properly, we need to display diffs # on every file in the union of both diffsets. Iterating over one diffset # or the other doesn't suffice. # # We build a list of parts containing the source filediff, the interdiff # filediff (if specified), and whether to force showing an interdiff # (in the case where a file existed in the source filediff but was # reverted in the interdiff). filediff_parts = [] for filediff in filediffs: interfilediff = None if filediff.source_file in interdiff_map: interfilediff = interdiff_map[filediff.source_file] del (interdiff_map[filediff.source_file]) filediff_parts.append((filediff, interfilediff, interdiffset != None)) if interdiffset: # We've removed everything in the map that we've already found. # What's left are interdiff files that are new. They have no file # to diff against. # # The end result is going to be a view that's the same as when you're # viewing a standard diff. As such, we can pretend the interdiff is # the source filediff and not specify an interdiff. Keeps things # simple, code-wise, since we really have no need to special-case # this. for interdiff in interdiff_map.values(): filediff_parts.append((interdiff, None, False)) files = [] index = 0 for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts filediff_revision_str = get_revision_str(filediff.source_revision) newfile = filediff.source_revision == PRE_CREATION if interdiffset: # First, find out if we want to even process this one. # We only process if there's a difference in files. if filediff and interfilediff and filediff.diff == interfilediff.diff: continue source_revision = "Diff Revision %s" % diffset.revision if not interfilediff and force_interdiff: dest_revision = "Diff Revision %s - File Reverted" % interdiffset.revision else: dest_revision = "Diff Revision %s" % interdiffset.revision else: source_revision = get_revision_str(filediff.source_revision) if newfile: dest_revision = _("New File") else: dest_revision = _("New Change") i = filediff.source_file.rfind("/") if i != -1: basepath = filediff.source_file[:i] basename = filediff.source_file[i + 1 :] else: basepath = "" basename = filediff.source_file file = { "depot_filename": filediff.source_file, "basename": basename, "basepath": basepath, "revision": source_revision, "dest_revision": dest_revision, "filediff": filediff, "interfilediff": interfilediff, "force_interdiff": force_interdiff, "binary": filediff.binary, "newfile": newfile, "index": len(files), } if load_chunks: chunks = [] if not filediff.binary: key = key_prefix if not force_interdiff: key += str(filediff.id) elif interfilediff: key += "interdiff-%s-%s" % (filediff.id, interfilediff.id) else: key += "interdiff-%s-none" % filediff.id chunks = cache_memoize( key, lambda: get_chunks( filediff.diffset, filediff, interfilediff, force_interdiff, enable_syntax_highlighting ), large_data=True, ) file["chunks"] = chunks file["changed_chunks"] = [] file["whitespace_only"] = True for j, chunk in enumerate(file["chunks"]): chunk["index"] = j if chunk["change"] != "equal": file["changed_chunks"].append(chunk) meta = chunk.get("meta", {}) if not meta.get("whitespace_chunk", False): file["whitespace_only"] = False file["num_changes"] = len(file["changed_chunks"]) files.append(file) def cmp_file(x, y): # Sort based on basepath in asc order if x["basepath"] != y["basepath"]: return cmp(x["basepath"], y["basepath"]) # Sort based on filename in asc order, then basod on extension in desc # order, to make *.h be ahead of *.c/cpp x_file, x_ext = os.path.splitext(x["basename"]) y_file, y_ext = os.path.splitext(y["basename"]) if x_file != y_file: return cmp(x_file, y_file) else: return cmp(y_ext, x_ext) files.sort(cmp_file) log_timer.done() return files
def get_diff_files(diffset, filediff=None, interdiffset=None, request=None): """Generates a list of files that will be displayed in a diff. This will go through the given diffset/interdiffset, or a given filediff within that diffset, and generate the list of files that will be displayed. This file list will contain a bunch of metadata on the files, such as the index, original/modified names, revisions, associated filediffs/diffsets, and so on. This can be used along with populate_diff_chunks to build a full list containing all diff chunks used for rendering a side-by-side diff. """ if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id), request=request) else: log_timer = log_timed("Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id), request=request) else: filediffs = diffset.files.select_related().all() if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id), request=request) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id, request=request) # A map used to quickly look up the equivalent interfilediff given a # source file. interdiff_map = {} if interdiffset: for interfilediff in interdiffset.files.all(): if (not filediff or filediff.source_file == interfilediff.source_file): interdiff_map[interfilediff.source_file] = interfilediff # In order to support interdiffs properly, we need to display diffs # on every file in the union of both diffsets. Iterating over one diffset # or the other doesn't suffice. # # We build a list of parts containing the source filediff, the interdiff # filediff (if specified), and whether to force showing an interdiff # (in the case where a file existed in the source filediff but was # reverted in the interdiff). has_interdiffset = interdiffset is not None filediff_parts = [(temp_filediff, interdiff_map.pop(temp_filediff.source_file, None), has_interdiffset) for temp_filediff in filediffs] if interdiffset: # We've removed everything in the map that we've already found. # What's left are interdiff files that are new. They have no file # to diff against. # # The end result is going to be a view that's the same as when you're # viewing a standard diff. As such, we can pretend the interdiff is # the source filediff and not specify an interdiff. Keeps things # simple, code-wise, since we really have no need to special-case # this. filediff_parts += [(interdiff, None, False) for interdiff in interdiff_map.itervalues()] files = [] for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts newfile = (filediff.source_revision == PRE_CREATION) if interdiffset: # First, find out if we want to even process this one. # We only process if there's a difference in files. if (filediff and interfilediff and filediff.diff == interfilediff.diff): continue source_revision = _("Diff Revision %s") % diffset.revision if not interfilediff and force_interdiff: dest_revision = _("Diff Revision %s - File Reverted") % \ interdiffset.revision else: dest_revision = _("Diff Revision %s") % interdiffset.revision else: source_revision = get_revision_str(filediff.source_revision) if newfile: dest_revision = _("New File") else: dest_revision = _("New Change") i = filediff.source_file.rfind('/') if i != -1: basepath = filediff.source_file[:i] basename = filediff.source_file[i + 1:] else: basepath = "" basename = filediff.source_file tool = filediff.diffset.repository.get_scmtool() depot_filename = tool.normalize_path_for_display(filediff.source_file) dest_filename = tool.normalize_path_for_display(filediff.dest_file) files.append({ 'depot_filename': depot_filename, 'dest_filename': dest_filename or depot_filename, 'basename': basename, 'basepath': basepath, 'revision': source_revision, 'dest_revision': dest_revision, 'filediff': filediff, 'interfilediff': interfilediff, 'force_interdiff': force_interdiff, 'binary': filediff.binary, 'deleted': filediff.deleted, 'moved': filediff.moved, 'newfile': newfile, 'index': len(files), 'chunks_loaded': False, 'is_new_file': (newfile and not interfilediff and not filediff.parent_diff), }) def cmp_file(x, y): # Sort based on basepath in asc order if x["basepath"] != y["basepath"]: return cmp(x["basepath"], y["basepath"]) # Sort based on filename in asc order, then based on extension in desc # order, to make *.h be ahead of *.c/cpp x_file, x_ext = os.path.splitext(x["basename"]) y_file, y_ext = os.path.splitext(y["basename"]) if x_file != y_file: return cmp(x_file, y_file) else: return cmp(y_ext, x_ext) files.sort(cmp_file) log_timer.done() return files
def get_chunks(diffset, filediff, interfilediff, force_interdiff, enable_syntax_highlighting): def diff_line(vlinenum, oldlinenum, newlinenum, oldline, newline, oldmarkup, newmarkup): # This function accesses the variable meta, defined in an outer context. if oldline and newline and oldline != newline: oldregion, newregion = get_line_changed_regions(oldline, newline) else: oldregion = newregion = [] result = [vlinenum, oldlinenum or '', mark_safe(oldmarkup or ''), oldregion, newlinenum or '', mark_safe(newmarkup or ''), newregion, (oldlinenum, newlinenum) in meta['whitespace_lines']] if oldlinenum and oldlinenum in meta.get('moved', {}): destination = meta["moved"][oldlinenum] result.append(destination) elif newlinenum and newlinenum in meta.get('moved', {}): destination = meta["moved"][newlinenum] result.append(destination) return result def new_chunk(lines, start, end, collapsable=False, tag='equal', meta=None): if not meta: meta = {} left_headers = list(get_interesting_headers(differ, lines, start, end - 1, False)) right_headers = list(get_interesting_headers(differ, lines, start, end - 1, True)) meta['left_headers'] = left_headers meta['right_headers'] = right_headers if left_headers: last_header[0] = left_headers[-1][1] if right_headers: last_header[1] = right_headers[-1][1] if (collapsable and end < len(lines) and (last_header[0] or last_header[1])): meta['headers'] = [ (last_header[0] or "").strip(), (last_header[1] or "").strip(), ] return { 'lines': lines[start:end], 'numlines': end - start, 'change': tag, 'collapsable': collapsable, 'meta': meta, } def get_interesting_headers(differ, lines, start, end, is_modified_file): """Returns all headers for a region of a diff. This scans for all headers that fall within the specified range of the specified lines on both the original and modified files. """ possible_functions = differ.get_interesting_lines('header', is_modified_file) if not possible_functions: raise StopIteration try: if is_modified_file: last_index = last_header_index[1] i1 = lines[start][4] i2 = lines[end - 1][4] else: last_index = last_header_index[0] i1 = lines[start][1] i2 = lines[end - 1][1] except IndexError: raise StopIteration for i in xrange(last_index, len(possible_functions)): linenum, line = possible_functions[i] linenum += 1 if linenum > i2: break elif linenum >= i1: last_index = i yield (linenum, line) if is_modified_file: last_header_index[1] = last_index else: last_header_index[0] = last_index def apply_pygments(data, filename): # XXX Guessing is preferable but really slow, especially on XML # files. #if filename.endswith(".xml"): lexer = get_lexer_for_filename(filename, stripnl=False, encoding='utf-8') #else: # lexer = guess_lexer_for_filename(filename, data, stripnl=False) try: # This is only available in 0.7 and higher lexer.add_filter('codetagify') except AttributeError: pass return pygments.highlight(data, lexer, NoWrapperHtmlFormatter()).splitlines() # There are three ways this function is called: # # 1) filediff, no interfilediff # - Returns chunks for a single filediff. This is the usual way # people look at diffs in the diff viewer. # # In this mode, we get the original file based on the filediff # and then patch it to get the resulting file. # # This is also used for interdiffs where the source revision # has no equivalent modified file but the interdiff revision # does. It's no different than a standard diff. # # 2) filediff, interfilediff # - Returns chunks showing the changes between a source filediff # and the interdiff. # # This is the typical mode used when showing the changes # between two diffs. It requires that the file is included in # both revisions of a diffset. # # 3) filediff, no interfilediff, force_interdiff # - Returns chunks showing the changes between a source # diff and an unmodified version of the diff. # # This is used when the source revision in the diffset contains # modifications to a file which have then been reverted in the # interdiff revision. We don't actually have an interfilediff # in this case, so we have to indicate that we are indeed in # interdiff mode so that we can special-case this and not # grab a patched file for the interdiff version. assert filediff file = filediff.source_file old = get_original_file(filediff) new = get_patched_file(old, filediff) if interfilediff: old = new interdiff_orig = get_original_file(interfilediff) new = get_patched_file(interdiff_orig, interfilediff) elif force_interdiff: # Basically, revert the change. old, new = new, old encoding = diffset.repository.encoding or 'iso-8859-15' old = convert_to_utf8(old, encoding) new = convert_to_utf8(new, encoding) # Normalize the input so that if there isn't a trailing newline, we add # it. if old and old[-1] != '\n': old += '\n' if new and new[-1] != '\n': new += '\n' a = NEWLINES_RE.split(old or '') b = NEWLINES_RE.split(new or '') # Remove the trailing newline, now that we've split this. This will # prevent a duplicate line number at the end of the diff. del(a[-1]) del(b[-1]) a_num_lines = len(a) b_num_lines = len(b) markup_a = markup_b = None siteconfig = SiteConfiguration.objects.get_current() threshold = siteconfig.get('diffviewer_syntax_highlighting_threshold') if threshold and (a_num_lines > threshold or b_num_lines > threshold): enable_syntax_highlighting = False if enable_syntax_highlighting: repository = filediff.diffset.repository tool = repository.get_scmtool() source_file = tool.normalize_path_for_display(filediff.source_file) dest_file = tool.normalize_path_for_display(filediff.dest_file) try: # TODO: Try to figure out the right lexer for these files # once instead of twice. markup_a = apply_pygments(old or '', source_file) markup_b = apply_pygments(new or '', dest_file) except: pass if not markup_a: markup_a = NEWLINES_RE.split(escape(old)) if not markup_b: markup_b = NEWLINES_RE.split(escape(new)) linenum = 1 last_header = [None, None] last_header_index = [0, 0] ignore_space = True for pattern in siteconfig.get("diffviewer_include_space_patterns"): if fnmatch.fnmatch(file, pattern): ignore_space = False break differ = Differ(a, b, ignore_space=ignore_space, compat_version=diffset.diffcompat) # Register any regexes for interesting lines we may want to show. register_interesting_lines_for_filename(differ, file) # TODO: Make this back into a preference if people really want it. context_num_lines = siteconfig.get("diffviewer_context_num_lines") collapse_threshold = 2 * context_num_lines + 3 if interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (filediff.id, interfilediff.id, filediff.source_file)) else: log_timer = log_timed( "Generating diff chunks for filediff id %s (%s)" % (filediff.id, filediff.source_file)) for tag, i1, i2, j1, j2, meta in opcodes_with_metadata(differ): oldlines = markup_a[i1:i2] newlines = markup_b[j1:j2] numlines = max(len(oldlines), len(newlines)) lines = map(diff_line, xrange(linenum, linenum + numlines), xrange(i1 + 1, i2 + 1), xrange(j1 + 1, j2 + 1), a[i1:i2], b[j1:j2], oldlines, newlines) if tag == 'equal' and numlines > collapse_threshold: last_range_start = numlines - context_num_lines if linenum == 1: yield new_chunk(lines, 0, last_range_start, True) yield new_chunk(lines, last_range_start, numlines) else: yield new_chunk(lines, 0, context_num_lines) if i2 == a_num_lines and j2 == b_num_lines: yield new_chunk(lines, context_num_lines, numlines, True) else: yield new_chunk(lines, context_num_lines, last_range_start, True) yield new_chunk(lines, last_range_start, numlines) else: yield new_chunk(lines, 0, numlines, False, tag, meta) linenum += numlines log_timer.done()
def patch(diff, orig_file, filename, request=None): """Apply a diff to a file. This delegates out to ``patch`` because noone except Larry Wall knows how to patch. Args: diff (bytes): The contents of the diff to apply. orig_file (bytes): The contents of the original file. filename (unicode): The name of the file being patched. request (django.http.HttpRequest, optional): The HTTP request, for use in logging. Returns: bytes: The contents of the patched file. Raises: reviewboard.diffutils.errors.PatchError: An error occurred when trying to apply the patch. """ log_timer = log_timed('Patching file %s' % filename, request=request) if not diff.strip(): # Someone uploaded an unchanged file. Return the one we're patching. return orig_file # Prepare the temporary directory if none is available tempdir = tempfile.mkdtemp(prefix='reviewboard.') try: orig_file = convert_line_endings(orig_file) diff = convert_line_endings(diff) (fd, oldfile) = tempfile.mkstemp(dir=tempdir) f = os.fdopen(fd, 'w+b') f.write(orig_file) f.close() newfile = '%s-new' % oldfile process = subprocess.Popen(['patch', '-o', newfile, oldfile], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=tempdir) with controlled_subprocess('patch', process) as p: stdout, stderr = p.communicate(diff) failure = p.returncode try: with open(newfile, 'r') as f: new_file = f.read() except Exception: new_file = None if failure: rejects_file = '%s.rej' % newfile try: with open(rejects_file, 'rb') as f: rejects = f.read() except Exception: rejects = None error_output = stderr.strip() or stdout.strip() # Munge the output to show the filename instead of # randomly-generated tempdir locations. base_filename = os.path.basename(filename) error_output = ( error_output .replace(rejects_file, '%s.rej' % base_filename) .replace(oldfile, base_filename) ) raise PatchError(filename, error_output, orig_file, new_file, diff, rejects) return new_file finally: shutil.rmtree(tempdir) log_timer.done()
def get_chunks_uncached(self): """Yield the list of chunks, bypassing the cache.""" old = get_original_file(self.filediff, self.request, self.encoding_list) new = get_patched_file(old, self.filediff, self.request) if self.base_filediff is not None: # The diff is against a commit that: # # 1. Follows the first commit in a series (the first won't have # a base_commit/base_filediff that can be looked up) # # 2. Follows a commit that modifies this file, or is the base # commit that modifies this file. # # We'll be diffing against the patched version of this commit's # version of the file. old = get_original_file(self.base_filediff, self.request, self.encoding_list) old = get_patched_file(old, self.base_filediff, self.request) elif self.filediff.commit_id: # This diff is against a commit, but no previous FileDiff # modifying this file could be found. As per the above comment, # this could end up being the very first commit in a series, or # it might not have been modified in the base commit or any # previous commit. # # We'll need to fetch the first ancestor of this file in the # commit history, if we can find one. We'll base the "old" version # of the file on the original version of this commit, meaning that # this commit and all modifications since will be shown as "new". # Basically, viewing the upstream of the file, before any commits. # # This should be safe because, without a base_filediff, there # should be no older commit containing modifications that we want # to diff against. This would be the first one, and we're using # its upstream changes. ancestors = self.filediff.get_ancestors(minimal=True) if ancestors: old = get_original_file(ancestors[0], self.request, self.encoding_list) if self.filediff.orig_sha1 is None: self.filediff.extra_data.update({ 'orig_sha1': self._get_checksum(old), 'patched_sha1': self._get_checksum(new), }) self.filediff.save(update_fields=['extra_data']) if self.interfilediff: old = new interdiff_orig = get_original_file(self.interfilediff, self.request, self.encoding_list) new = get_patched_file(interdiff_orig, self.interfilediff, self.request) if self.interfilediff.orig_sha1 is None: self.interfilediff.extra_data.update({ 'orig_sha1': self._get_checksum(interdiff_orig), 'patched_sha1': self._get_checksum(new), }) self.interfilediff.save(update_fields=['extra_data']) elif self.force_interdiff: # Basically, revert the change. old, new = new, old if self.interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (self.filediff.id, self.interfilediff.id, self.filediff.source_file), request=self.request) else: log_timer = log_timed( "Generating diff chunks for self.filediff id %s (%s)" % (self.filediff.id, self.filediff.source_file), request=self.request) for chunk in self.generate_chunks(old, new): yield chunk log_timer.done() if (not self.interfilediff and not self.base_filediff and not self.force_interdiff): insert_count = self.counts['insert'] delete_count = self.counts['delete'] replace_count = self.counts['replace'] equal_count = self.counts['equal'] self.filediff.set_line_counts( insert_count=insert_count, delete_count=delete_count, replace_count=replace_count, equal_count=equal_count, total_line_count=(insert_count + delete_count + replace_count + equal_count))
def get_diff_files(diffset, filediff=None, interdiffset=None, enable_syntax_highlighting=True, load_chunks=True): if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id)) else: log_timer = log_timed("Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id)) else: filediffs = diffset.files.select_related().all() if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id)) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id) # A map used to quickly look up the equivalent interfilediff given a # source file. interdiff_map = {} if interdiffset: for interfilediff in interdiffset.files.all(): if not filediff or \ filediff.source_file == interfilediff.source_file: interdiff_map[interfilediff.source_file] = interfilediff key_prefix = "diff-sidebyside-" if enable_syntax_highlighting: key_prefix += "hl-" # In order to support interdiffs properly, we need to display diffs # on every file in the union of both diffsets. Iterating over one diffset # or the other doesn't suffice. # # We build a list of parts containing the source filediff, the interdiff # filediff (if specified), and whether to force showing an interdiff # (in the case where a file existed in the source filediff but was # reverted in the interdiff). filediff_parts = [] for filediff in filediffs: interfilediff = None if filediff.source_file in interdiff_map: interfilediff = interdiff_map[filediff.source_file] del(interdiff_map[filediff.source_file]) filediff_parts.append((filediff, interfilediff, interdiffset != None)) if interdiffset: # We've removed everything in the map that we've already found. # What's left are interdiff files that are new. They have no file # to diff against. # # The end result is going to be a view that's the same as when you're # viewing a standard diff. As such, we can pretend the interdiff is # the source filediff and not specify an interdiff. Keeps things # simple, code-wise, since we really have no need to special-case # this. filediff_parts += [(interdiff, None, False) for interdiff in interdiff_map.values()] files = [] for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts newfile = (filediff.source_revision == PRE_CREATION) if interdiffset: # First, find out if we want to even process this one. # We only process if there's a difference in files. if (filediff and interfilediff and filediff.diff == interfilediff.diff): continue source_revision = "Diff Revision %s" % diffset.revision if not interfilediff and force_interdiff: dest_revision = "Diff Revision %s - File Reverted" % \ interdiffset.revision else: dest_revision = "Diff Revision %s" % interdiffset.revision else: source_revision = get_revision_str(filediff.source_revision) if newfile: dest_revision = NEW_FILE_STR else: dest_revision = NEW_CHANGE_STR i = filediff.source_file.rfind('/') if i != -1: basepath = filediff.source_file[:i] basename = filediff.source_file[i + 1:] else: basepath = "" basename = filediff.source_file tool = filediff.diffset.repository.get_scmtool() depot_filename = tool.normalize_path_for_display(filediff.source_file) dest_filename = tool.normalize_path_for_display(filediff.dest_file) file = { 'depot_filename': depot_filename, 'dest_filename': dest_filename or depot_filename, 'basename': basename, 'basepath': basepath, 'revision': source_revision, 'dest_revision': dest_revision, 'filediff': filediff, 'interfilediff': interfilediff, 'force_interdiff': force_interdiff, 'binary': filediff.binary, 'deleted': filediff.deleted, 'newfile': newfile, 'index': len(files), } if load_chunks: chunks = [] if not filediff.binary and not filediff.deleted: key = key_prefix if not force_interdiff: key += str(filediff.id) elif interfilediff: key += "interdiff-%s-%s" % (filediff.id, interfilediff.id) else: key += "interdiff-%s-none" % filediff.id chunks = cache_memoize( key, lambda: list(get_chunks(filediff.diffset, filediff, interfilediff, force_interdiff, enable_syntax_highlighting)), large_data=True) file['chunks'] = chunks file['changed_chunk_indexes'] = [] file['whitespace_only'] = True for j, chunk in enumerate(file['chunks']): chunk['index'] = j if chunk['change'] != 'equal': file['changed_chunk_indexes'].append(j) meta = chunk.get('meta', {}) if not meta.get('whitespace_chunk', False): file['whitespace_only'] = False file['num_changes'] = len(file['changed_chunk_indexes']) files.append(file) def cmp_file(x, y): # Sort based on basepath in asc order if x["basepath"] != y["basepath"]: return cmp(x["basepath"], y["basepath"]) # Sort based on filename in asc order, then based on extension in desc # order, to make *.h be ahead of *.c/cpp x_file, x_ext = os.path.splitext(x["basename"]) y_file, y_ext = os.path.splitext(y["basename"]) if x_file != y_file: return cmp(x_file, y_file) else: return cmp(y_ext, x_ext) files.sort(cmp_file) log_timer.done() return files
def patch(diff, file, filename, request=None): """Apply a diff to a file. Delegates out to `patch` because noone except Larry Wall knows how to patch.""" log_timer = log_timed("Patching file %s" % filename, request=request) if not diff.strip(): # Someone uploaded an unchanged file. Return the one we're patching. return file # Prepare the temporary directory if none is available tempdir = tempfile.mkdtemp(prefix="reviewboard.") (fd, oldfile) = tempfile.mkstemp(dir=tempdir) f = os.fdopen(fd, "w+b") f.write(convert_line_endings(file)) f.close() diff = convert_line_endings(diff) newfile = "%s-new" % oldfile process = subprocess.Popen( ["patch", "-o", newfile, oldfile], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=tempdir, ) with controlled_subprocess("patch", process) as p: stdout, stderr = p.communicate(diff) failure = p.returncode if failure: absolute_path = os.path.join(tempdir, os.path.basename(filename)) with open("%s.diff" % absolute_path, "w") as f: f.write(diff) log_timer.done() # FIXME: This doesn't provide any useful error report on why the patch # failed to apply, which makes it hard to debug. We might also want to # have it clean up if DEBUG=False raise Exception( _( "The patch to '%(filename)s' didn't apply cleanly. The " "temporary files have been left in '%(tempdir)s' for debugging " "purposes.\n" "`patch` returned: %(output)s" ) % {"filename": filename, "tempdir": tempdir, "output": stderr} ) with open(newfile, "r") as f: data = f.read() os.unlink(oldfile) os.unlink(newfile) os.rmdir(tempdir) log_timer.done() return data
def get_diff_files(diffset, filediff=None, interdiffset=None, request=None): """Generates a list of files that will be displayed in a diff. This will go through the given diffset/interdiffset, or a given filediff within that diffset, and generate the list of files that will be displayed. This file list will contain a bunch of metadata on the files, such as the index, original/modified names, revisions, associated filediffs/diffsets, and so on. This can be used along with populate_diff_chunks to build a full list containing all diff chunks used for rendering a side-by-side diff. """ if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id), request=request) else: log_timer = log_timed("Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id), request=request) else: filediffs = diffset.files.select_related().all() if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id), request=request) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id, request=request) # A map used to quickly look up the equivalent interfilediff given a # source file. interdiff_map = {} if interdiffset: for interfilediff in interdiffset.files.all(): if (not filediff or filediff.source_file == interfilediff.source_file): interdiff_map[interfilediff.source_file] = interfilediff # In order to support interdiffs properly, we need to display diffs # on every file in the union of both diffsets. Iterating over one diffset # or the other doesn't suffice. # # We build a list of parts containing the source filediff, the interdiff # filediff (if specified), and whether to force showing an interdiff # (in the case where a file existed in the source filediff but was # reverted in the interdiff). has_interdiffset = interdiffset is not None filediff_parts = [ (temp_filediff, interdiff_map.pop(temp_filediff.source_file, None), has_interdiffset) for temp_filediff in filediffs ] if interdiffset: # We've removed everything in the map that we've already found. # What's left are interdiff files that are new. They have no file # to diff against. # # The end result is going to be a view that's the same as when you're # viewing a standard diff. As such, we can pretend the interdiff is # the source filediff and not specify an interdiff. Keeps things # simple, code-wise, since we really have no need to special-case # this. filediff_parts += [ (interdiff, None, False) for interdiff in six.itervalues(interdiff_map) ] files = [] for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts newfile = filediff.is_new if interdiffset: # First, find out if we want to even process this one. # We only process if there's a difference in files. if (filediff and interfilediff and filediff.diff == interfilediff.diff): continue source_revision = _("Diff Revision %s") % diffset.revision if not interfilediff and force_interdiff: dest_revision = (_("Diff Revision %s - File Reverted") % interdiffset.revision) else: dest_revision = _("Diff Revision %s") % interdiffset.revision else: source_revision = get_revision_str(filediff.source_revision) if newfile: dest_revision = _("New File") else: dest_revision = _("New Change") tool = filediff.diffset.repository.get_scmtool() depot_filename = tool.normalize_path_for_display(filediff.source_file) dest_filename = tool.normalize_path_for_display(filediff.dest_file) f = { 'depot_filename': depot_filename, 'dest_filename': dest_filename or depot_filename, 'revision': source_revision, 'dest_revision': dest_revision, 'filediff': filediff, 'interfilediff': interfilediff, 'force_interdiff': force_interdiff, 'binary': filediff.binary, 'deleted': filediff.deleted, 'moved': filediff.moved, 'copied': filediff.copied, 'moved_or_copied': filediff.moved or filediff.copied, 'newfile': newfile, 'index': len(files), 'chunks_loaded': False, 'is_new_file': (newfile and not interfilediff and not filediff.parent_diff), } if force_interdiff: f['force_interdiff_revision'] = interdiffset.revision files.append(f) log_timer.done() return get_sorted_filediffs(files, key=lambda f: f['filediff'])
def get_diff_files(diffset, filediff=None, interdiffset=None, interfilediff=None, request=None): """Return a list of files that will be displayed in a diff. This will go through the given diffset/interdiffset, or a given filediff within that diffset, and generate the list of files that will be displayed. This file list will contain a bunch of metadata on the files, such as the index, original/modified names, revisions, associated filediffs/diffsets, and so on. This can be used along with :py:func:`populate_diff_chunks` to build a full list containing all diff chunks used for rendering a side-by-side diff. Args: diffset (reviewboard.diffviewer.models.DiffSet): The diffset containing the files to return. filediff (reviewboard.diffviewer.models.FileDiff, optional): A specific file in the diff to return information for. interdiffset (reviewboard.diffviewer.models.DiffSet, optional): A second diffset used for an interdiff range. interfilediff (reviewboard.diffviewer.models.FileDiff, optional): A second specific file in ``interdiffset`` used to return information for. This should be provided if ``filediff`` and ``interdiffset`` are both provided. If it's ``None`` in this case, then the diff will be shown as reverted for this file. Returns: list of dict: A list of dictionaries containing information on the files to show in the diff, in the order in which they would be shown. """ if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed( "Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id), request=request, ) else: log_timer = log_timed( "Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id), request=request, ) else: filediffs = list(diffset.files.select_related().all()) if interdiffset: log_timer = log_timed( "Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id), request=request, ) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id, request=request) # Filediffs that were created with leading slashes stripped won't match # those created with them present, so we need to compare them without in # order for the filenames to match up properly. tool = diffset.repository.get_scmtool() if interdiffset: if not filediff: interfilediffs = list(interdiffset.files.all()) elif interfilediff: interfilediffs = [interfilediff] else: interfilediffs = [] filediff_parts = [] matched_filediffs = get_matched_interdiff_files(tool=tool, filediffs=filediffs, interfilediffs=interfilediffs) for temp_filediff, temp_interfilediff in matched_filediffs: if temp_filediff: filediff_parts.append((temp_filediff, temp_interfilediff, True)) elif temp_interfilediff: filediff_parts.append((temp_interfilediff, None, False)) else: logging.error( "get_matched_interdiff_files returned an entry with an " "empty filediff and interfilediff for diffset=%r, " "interdiffset=%r, filediffs=%r, interfilediffs=%r", diffset, interdiffset, filediffs, interfilediffs, ) raise ValueError( "Internal error: get_matched_interdiff_files returned an " "entry with an empty filediff and interfilediff! Please " "report this along with information from the server " "error log." ) else: # We're not working with interdiffs. We can easily create the # filediff_parts directly. filediff_parts = [(temp_filediff, None, False) for temp_filediff in filediffs] # Now that we have all the bits and pieces we care about for the filediffs, # we can start building information about each entry on the diff viewer. files = [] for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts newfile = filediff.is_new if interdiffset: # First, find out if we want to even process this one. # If the diffs are identical, or the patched files are identical, # or if the files were deleted in both cases, then we can be # absolutely sure that there's nothing interesting to show to # the user. if ( filediff and interfilediff and ( filediff.diff == interfilediff.diff or (filediff.deleted and interfilediff.deleted) or (filediff.patched_sha1 is not None and filediff.patched_sha1 == interfilediff.patched_sha1) ) ): continue source_revision = _("Diff Revision %s") % diffset.revision else: source_revision = get_revision_str(filediff.source_revision) if interfilediff: dest_revision = _("Diff Revision %s") % interdiffset.revision else: if force_interdiff: dest_revision = _("Diff Revision %s - File Reverted") % interdiffset.revision elif newfile: dest_revision = _("New File") else: dest_revision = _("New Change") if interfilediff: raw_depot_filename = filediff.dest_file raw_dest_filename = interfilediff.dest_file else: raw_depot_filename = filediff.source_file raw_dest_filename = filediff.dest_file depot_filename = tool.normalize_path_for_display(raw_depot_filename) dest_filename = tool.normalize_path_for_display(raw_dest_filename) f = { "depot_filename": depot_filename, "dest_filename": dest_filename or depot_filename, "revision": source_revision, "dest_revision": dest_revision, "filediff": filediff, "interfilediff": interfilediff, "force_interdiff": force_interdiff, "binary": filediff.binary, "deleted": filediff.deleted, "moved": filediff.moved, "copied": filediff.copied, "moved_or_copied": filediff.moved or filediff.copied, "newfile": newfile, "index": len(files), "chunks_loaded": False, "is_new_file": (newfile and not interfilediff and not filediff.parent_diff), } if force_interdiff: f["force_interdiff_revision"] = interdiffset.revision files.append(f) log_timer.done() if len(files) == 1: return files else: return get_sorted_filediffs(files, key=lambda f: f["interfilediff"] or f["filediff"])
def _get_chunks_uncached(self): """Returns the list of chunks, bypassing the cache.""" old = get_original_file(self.filediff, self.request) new = get_patched_file(old, self.filediff, self.request) if self.interfilediff: old = new interdiff_orig = get_original_file(self.interfilediff, self.request) new = get_patched_file(interdiff_orig, self.interfilediff, self.request) elif self.force_interdiff: # Basically, revert the change. old, new = new, old encoding = self.diffset.repository.encoding or 'iso-8859-15' old = self._convert_to_utf8(old, encoding) new = self._convert_to_utf8(new, encoding) # Normalize the input so that if there isn't a trailing newline, we add # it. if old and old[-1] != '\n': old += '\n' if new and new[-1] != '\n': new += '\n' a = self.NEWLINES_RE.split(old or '') b = self.NEWLINES_RE.split(new or '') # Remove the trailing newline, now that we've split this. This will # prevent a duplicate line number at the end of the diff. del a[-1] del b[-1] a_num_lines = len(a) b_num_lines = len(b) markup_a = markup_b = None if self._get_enable_syntax_highlighting(old, new, a, b): repository = self.filediff.diffset.repository tool = repository.get_scmtool() source_file = \ tool.normalize_path_for_display(self.filediff.source_file) dest_file = \ tool.normalize_path_for_display(self.filediff.dest_file) try: # TODO: Try to figure out the right lexer for these files # once instead of twice. markup_a = self._apply_pygments(old or '', source_file) markup_b = self._apply_pygments(new or '', dest_file) except: pass if not markup_a: markup_a = self.NEWLINES_RE.split(escape(old)) if not markup_b: markup_b = self.NEWLINES_RE.split(escape(new)) siteconfig = SiteConfiguration.objects.get_current() ignore_space = True for pattern in siteconfig.get('diffviewer_include_space_patterns'): if fnmatch.fnmatch(self.filename, pattern): ignore_space = False break self.differ = get_differ(a, b, ignore_space=ignore_space, compat_version=self.diffset.diffcompat) self.differ.add_interesting_lines_for_headers(self.filename) context_num_lines = siteconfig.get("diffviewer_context_num_lines") collapse_threshold = 2 * context_num_lines + 3 if self.interfilediff: log_timer = log_timed( "Generating diff chunks for interdiff ids %s-%s (%s)" % (self.filediff.id, self.interfilediff.id, self.filediff.source_file), request=self.request) else: log_timer = log_timed( "Generating diff chunks for self.filediff id %s (%s)" % (self.filediff.id, self.filediff.source_file), request=self.request) line_num = 1 opcodes_generator = get_diff_opcode_generator(self.differ, self.filediff, self.interfilediff) for tag, i1, i2, j1, j2, meta in opcodes_generator: old_lines = markup_a[i1:i2] new_lines = markup_b[j1:j2] num_lines = max(len(old_lines), len(new_lines)) self._cur_meta = meta lines = map(self._diff_line, range(line_num, line_num + num_lines), range(i1 + 1, i2 + 1), range(j1 + 1, j2 + 1), a[i1:i2], b[j1:j2], old_lines, new_lines) self._cur_meta = None if tag == 'equal' and num_lines > collapse_threshold: last_range_start = num_lines - context_num_lines if line_num == 1: yield self._new_chunk(lines, 0, last_range_start, True) yield self._new_chunk(lines, last_range_start, num_lines) else: yield self._new_chunk(lines, 0, context_num_lines) if i2 == a_num_lines and j2 == b_num_lines: yield self._new_chunk(lines, context_num_lines, num_lines, True) else: yield self._new_chunk(lines, context_num_lines, last_range_start, True) yield self._new_chunk(lines, last_range_start, num_lines) else: yield self._new_chunk(lines, 0, num_lines, False, tag, meta) line_num += num_lines log_timer.done()
def get_diff_files(diffset, filediff=None, interdiffset=None, request=None): """Generates a list of files that will be displayed in a diff. This will go through the given diffset/interdiffset, or a given filediff within that diffset, and generate the list of files that will be displayed. This file list will contain a bunch of metadata on the files, such as the index, original/modified names, revisions, associated filediffs/diffsets, and so on. This can be used along with populate_diff_chunks to build a full list containing all diff chunks used for rendering a side-by-side diff. """ if filediff: filediffs = [filediff] if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s, filediff %s" % (diffset.id, interdiffset.id, filediff.id), request=request) else: log_timer = log_timed("Generating diff file info for " "diffset id %s, filediff %s" % (diffset.id, filediff.id), request=request) else: filediffs = diffset.files.select_related().all() if interdiffset: log_timer = log_timed("Generating diff file info for " "interdiffset ids %s-%s" % (diffset.id, interdiffset.id), request=request) else: log_timer = log_timed("Generating diff file info for " "diffset id %s" % diffset.id, request=request) # A map used to quickly look up the equivalent interfilediff given a # source file. interdiff_map = {} # Filediffs that were created with leading slashes stripped won't match # those created with them present, so we need to compare them without in # order for the filenames to match up properly. parser = diffset.repository.get_scmtool().get_parser('') def _normfile(filename): return parser.normalize_diff_filename(filename) if interdiffset: for interfilediff in interdiffset.files.all(): interfilediff_source_file = _normfile(interfilediff.source_file) if (not filediff or _normfile(filediff.source_file) == interfilediff_source_file): interdiff_map[interfilediff_source_file] = interfilediff # In order to support interdiffs properly, we need to display diffs # on every file in the union of both diffsets. Iterating over one diffset # or the other doesn't suffice. # # We build a list of parts containing the source filediff, the interdiff # filediff (if specified), and whether to force showing an interdiff # (in the case where a file existed in the source filediff but was # reverted in the interdiff). has_interdiffset = interdiffset is not None filediff_parts = [(temp_filediff, interdiff_map.pop(_normfile(temp_filediff.source_file), None), has_interdiffset) for temp_filediff in filediffs] if interdiffset: # We've removed everything in the map that we've already found. # What's left are interdiff files that are new. They have no file # to diff against. # # The end result is going to be a view that's the same as when you're # viewing a standard diff. As such, we can pretend the interdiff is # the source filediff and not specify an interdiff. Keeps things # simple, code-wise, since we really have no need to special-case # this. filediff_parts += [(interdiff, None, False) for interdiff in six.itervalues(interdiff_map)] files = [] for parts in filediff_parts: filediff, interfilediff, force_interdiff = parts newfile = filediff.is_new if interdiffset: # First, find out if we want to even process this one. # If the diffs are identical, or the patched files are identical, # or if the files were deleted in both cases, then we can be # absolutely sure that there's nothing interesting to show to # the user. if (filediff and interfilediff and (filediff.diff == interfilediff.diff or (filediff.deleted and interfilediff.deleted) or (filediff.patched_sha1 is not None and filediff.patched_sha1 == interfilediff.patched_sha1))): continue source_revision = _("Diff Revision %s") % diffset.revision if not interfilediff and force_interdiff: dest_revision = (_("Diff Revision %s - File Reverted") % interdiffset.revision) else: dest_revision = _("Diff Revision %s") % interdiffset.revision else: source_revision = get_revision_str(filediff.source_revision) if newfile: dest_revision = _("New File") else: dest_revision = _("New Change") tool = filediff.diffset.repository.get_scmtool() depot_filename = tool.normalize_path_for_display(filediff.source_file) dest_filename = tool.normalize_path_for_display(filediff.dest_file) f = { 'depot_filename': depot_filename, 'dest_filename': dest_filename or depot_filename, 'revision': source_revision, 'dest_revision': dest_revision, 'filediff': filediff, 'interfilediff': interfilediff, 'force_interdiff': force_interdiff, 'binary': filediff.binary, 'deleted': filediff.deleted, 'moved': filediff.moved, 'copied': filediff.copied, 'moved_or_copied': filediff.moved or filediff.copied, 'newfile': newfile, 'index': len(files), 'chunks_loaded': False, 'is_new_file': (newfile and not interfilediff and not filediff.parent_diff), } if force_interdiff: f['force_interdiff_revision'] = interdiffset.revision files.append(f) log_timer.done() return get_sorted_filediffs(files, key=lambda f: f['filediff'])