def make_filediff(self, is_new=False, diffset_history=None, diffset_revision=1, source_filename='file1', dest_filename='file2'): """Create and return a FileDiff with the given data.""" if is_new: source_revision = PRE_CREATION dest_revision = '' else: source_revision = '1' dest_revision = '2' repository = self.create_repository() if not diffset_history: diffset_history = DiffSetHistory.objects.create(name='testhistory') diffset = DiffSet.objects.create(name='test', revision=diffset_revision, repository=repository, history=diffset_history) filediff = FileDiff(source_file=source_filename, source_revision=source_revision, dest_file=dest_filename, dest_detail=dest_revision, diffset=diffset, binary=True) filediff.save() return filediff
def make_filediff( self, is_new=False, diffset_history=None, diffset_revision=1, source_filename="file1", dest_filename="file2" ): """Create and return a FileDiff with the given data.""" if is_new: source_revision = PRE_CREATION dest_revision = "" else: source_revision = "1" dest_revision = "2" repository = self.create_repository() if not diffset_history: diffset_history = DiffSetHistory.objects.create(name="testhistory") diffset = DiffSet.objects.create( name="test", revision=diffset_revision, repository=repository, history=diffset_history ) filediff = FileDiff( source_file=source_filename, source_revision=source_revision, dest_file=dest_filename, dest_detail=dest_revision, diffset=diffset, binary=True, ) filediff.save() return filediff
def make_filediff(self, is_new=False, diffset_history=None, diffset_revision=1, source_filename='file1', dest_filename='file2'): if is_new: source_revision = PRE_CREATION dest_revision = '' else: source_revision = '1' dest_revision = '2' repository = self.create_repository() if not diffset_history: diffset_history = DiffSetHistory.objects.create(name='testhistory') diffset = DiffSet.objects.create(name='test', revision=diffset_revision, repository=repository, history=diffset_history) filediff = FileDiff(source_file=source_filename, source_revision=source_revision, dest_file=dest_filename, dest_detail=dest_revision, diffset=diffset, binary=True) filediff.save() return filediff
def create(self, diff_file, parent_diff_file=None, diffset_history=None): tool = self.repository.get_scmtool() # Grab the base directory if there is one. if not tool.get_diffs_use_absolute_paths(): try: basedir = smart_unicode(self.cleaned_data['basedir']) except AttributeError: raise NoBaseDirError(_('The "Base Diff Path" field is required')) else: basedir = '' # Parse the diff files = list(self._process_files( diff_file, basedir, check_existance=(parent_diff_file is not None))) if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} if parent_diff_file: # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_diff_file, basedir, check_existance=True): parent_files[f.origFile] = f diffset = DiffSet(name=diff_file.name, revision=0, history=diffset_history, diffcompat=DEFAULT_DIFF_COMPAT_VERSION) diffset.repository = self.repository diffset.save() for f in files: if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data source_rev = parent_file.origInfo else: parent_content = "" source_rev = f.origInfo dest_file = os.path.join(basedir, f.newFile).replace("\\", "/") filediff = FileDiff(diffset=diffset, source_file=f.origFile, dest_file=dest_file, source_revision=smart_unicode(source_rev), dest_detail=f.newInfo, diff=f.data, parent_diff=parent_content, binary=f.binary) filediff.save() return diffset
def create_filediff(self, diffset, source_file='/test-file', dest_file='/test-file', source_revision='123', dest_detail='124', status=FileDiff.MODIFIED, diff=DEFAULT_FILEDIFF_DATA, save=True): """Create a FileDiff for testing. The FileDiff is tied to the given DiffSet. It's populated with default data that can be overridden by the caller. Args: diffset (reviewboard.diffviewer.models.DiffSet): The parent diff set that will own this file. source_file (unicode, optional): The source filename. dest_file (unicode, optional): The destination filename, which will be the same as ``source_file`` unless the file was moved/renamed/copied. source_revision (unicode, optional): The source revision. dest_detail (unicode, optional): The destination revision or other detail as found in the parsed diff. This may be a timestamp or some other value. status (unicode, optional): The status of the file. This is the operation performed as indicated in the diff. diff (bytes, optional): The diff contents. save (bool, optional): Whether to automatically save the resulting object. Returns: reviewboard.diffviewer.models.FileDiff: The resulting FileDiff. """ filediff = FileDiff(diffset=diffset, source_file=source_file, dest_file=dest_file, source_revision=source_revision, dest_detail=dest_detail, status=status, diff=diff) if save: filediff.save() return filediff
def create_filediff(self, diffset, source_file='/test-file', dest_file='/test-file', source_revision='123', dest_detail='124', status=FileDiff.MODIFIED, diff=DEFAULT_FILEDIFF_DATA, save=True): """Create a FileDiff for testing. The FileDiff is tied to the given DiffSet. It's populated with default data that can be overridden by the caller. Args: diffset (reviewboard.diffviewer.models.DiffSet): The parent diff set that will own this file. source_file (unicode, optional): The source filename. dest_file (unicode, optional): The destination filename, which will be the same as ``source_file`` unless the file was moved/renamed/copied. source_revision (unicode, optional): The source revision. dest_detail (unicode, optional): The destination revision or other detail as found in the parsed diff. This may be a timestamp or some other value. status (unicode, optional): The status of the file. This is the operation performed as indicated in the diff. diff (bytes, optional): The diff contents. save (bool, optional): Whether to automatically save the resulting object. Returns: reviewboard.diffviewer.models.FileDiff: The resulting FileDiff. """ filediff = FileDiff( diffset=diffset, source_file=source_file, dest_file=dest_file, source_revision=source_revision, dest_detail=dest_detail, status=status, diff=diff) if save: filediff.save() return filediff
def testLongFilenames(self): """Testing using long filenames (1024 characters) in FileDiff.""" long_filename = 'x' * 1024 repository = Repository.objects.get(pk=1) diffset = DiffSet.objects.create(name='test', revision=1, repository=repository) filediff = FileDiff(source_file=long_filename, dest_file='foo', diffset=diffset) filediff.save() filediff = FileDiff.objects.get(pk=filediff.id) self.assertEquals(filediff.source_file, long_filename)
def test_diff_hashes(self): """ Testing that uploading two of the same diff will result in only one database entry. """ repository = self.create_repository() diffset = DiffSet.objects.create(name='test', revision=1, repository=repository) with open(os.path.join(self.PREFIX, "diffs", "context", "foo.c.diff")) as f: data = f.read() filediff1 = FileDiff(diff=data, diffset=diffset) filediff1.save() filediff2 = FileDiff(diff=data, diffset=diffset) filediff2.save() self.assertEquals(filediff1.diff_hash, filediff2.diff_hash)
def create_from_data(self, repository, diff_file_name, diff_file_contents, parent_diff_file_name, parent_diff_file_contents, diffset_history, basedir, request, base_commit_id=None, save=True): """Create a DiffSet from raw diff data. The diff_file_contents and parent_diff_file_contents parameters are strings with the actual diff contents. """ from reviewboard.diffviewer.diffutils import convert_to_unicode from reviewboard.diffviewer.models import FileDiff tool = repository.get_scmtool() encoding, diff_text = convert_to_unicode( diff_file_contents, repository.get_encoding_list()) parser = tool.get_parser(diff_text) files = list(self._process_files( parser, basedir, repository, base_commit_id, request, check_existence=(not parent_diff_file_contents))) # Parse the diff if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} # This is used only for tools like Mercurial that use atomic changeset # IDs to identify all file versions but not individual file version # IDs. parent_commit_id = None if parent_diff_file_contents: diff_filenames = set([f.origFile for f in files]) parent_parser = tool.get_parser( convert_to_unicode(parent_diff_file_contents, [encoding])[1]) # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_parser, basedir, repository, base_commit_id, request, check_existence=True, limit_to=diff_filenames): parent_files[f.origFile] = f # This will return a non-None value only for tools that use # commit IDs to identify file versions as opposed to file revision # IDs. parent_commit_id = parent_parser.get_orig_commit_id() diffset = super(DiffSetManager, self).create( name=diff_file_name, revision=0, basedir=basedir, history=diffset_history, repository=repository, diffcompat=DiffCompatVersion.DEFAULT, base_commit_id=base_commit_id) if save: diffset.save() for f in files: if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data.encode(encoding) source_rev = parent_file.origInfo else: parent_content = b"" if parent_commit_id and f.origInfo != PRE_CREATION: source_rev = parent_commit_id else: source_rev = f.origInfo dest_file = os.path.join(basedir, f.newFile).replace("\\", "/") if f.deleted: status = FileDiff.DELETED elif f.moved: status = FileDiff.MOVED elif f.copied: status = FileDiff.COPIED else: status = FileDiff.MODIFIED filediff = FileDiff( diffset=diffset, source_file=parser.normalize_diff_filename(f.origFile), dest_file=parser.normalize_diff_filename(dest_file), source_revision=smart_unicode(source_rev), dest_detail=f.newInfo, diff=f.data.encode(encoding), parent_diff=parent_content, binary=f.binary, status=status) filediff.set_line_counts(raw_insert_count=f.insert_count, raw_delete_count=f.delete_count) if save: filediff.save() return diffset
def create_from_data(self, repository, diff_file_name, diff_file_contents, parent_diff_file_name, parent_diff_file_contents, diffset_history, basedir, request, base_commit_id=None, save=True): """Create a DiffSet from raw diff data. The diff_file_contents and parent_diff_file_contents parameters are strings with the actual diff contents. """ from reviewboard.diffviewer.diffutils import convert_to_unicode from reviewboard.diffviewer.models import FileDiff tool = repository.get_scmtool() encoding, diff_text = convert_to_unicode( diff_file_contents, repository.get_encoding_list()) parser = tool.get_parser(diff_text) files = list( self._process_files( parser, basedir, repository, base_commit_id, request, check_existence=(not parent_diff_file_contents))) # Parse the diff if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} # This is used only for tools like Mercurial that use atomic changeset # IDs to identify all file versions but not individual file version # IDs. parent_commit_id = None if parent_diff_file_contents: diff_filenames = set([f.origFile for f in files]) parent_parser = tool.get_parser( convert_to_unicode(parent_diff_file_contents, [encoding])[1]) # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_parser, basedir, repository, base_commit_id, request, check_existence=True, limit_to=diff_filenames): parent_files[f.origFile] = f # This will return a non-None value only for tools that use # commit IDs to identify file versions as opposed to file revision # IDs. parent_commit_id = parent_parser.get_orig_commit_id() diffset = super(DiffSetManager, self).create(name=diff_file_name, revision=0, basedir=basedir, history=diffset_history, repository=repository, diffcompat=DiffCompatVersion.DEFAULT, base_commit_id=base_commit_id) if save: diffset.save() for f in files: if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data.encode(encoding) source_rev = parent_file.origInfo else: parent_content = b"" if parent_commit_id and f.origInfo != PRE_CREATION: source_rev = parent_commit_id else: source_rev = f.origInfo dest_file = os.path.join(basedir, f.newFile).replace("\\", "/") if f.deleted: status = FileDiff.DELETED elif f.moved: status = FileDiff.MOVED elif f.copied: status = FileDiff.COPIED else: status = FileDiff.MODIFIED filediff = FileDiff(diffset=diffset, source_file=f.origFile, dest_file=dest_file, source_revision=smart_unicode(source_rev), dest_detail=f.newInfo, diff=f.data.encode(encoding), parent_diff=parent_content, binary=f.binary, status=status) filediff.set_line_counts(raw_insert_count=f.insert_count, raw_delete_count=f.delete_count) if save: filediff.save() return diffset
def create_from_data(self, repository, diff_file_name, diff_file_contents, parent_diff_file_name, parent_diff_file_contents, diffset_history, basedir, request, save=True): """Create a DiffSet from raw diff data. The diff_file_contents and parent_diff_file_contents parameters are strings with the actual diff contents. """ from reviewboard.diffviewer.models import FileDiff tool = repository.get_scmtool() files = list(self._process_files( diff_file_contents, basedir, repository, request, check_existence=(not parent_diff_file_contents))) # Parse the diff if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} # This is used only for tools like Mercurial that use atomic changeset # IDs to identify all file versions but not individual file version # IDs. parent_changeset_id = None if parent_diff_file_contents: diff_filenames = set([f.origFile for f in files]) # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_diff_file_contents, basedir, repository, request, check_existence=True, limit_to=diff_filenames): parent_files[f.origFile] = f # Store the original changeset ID if we have it; this should # be the same for all files. if f.origChangesetId: parent_changeset_id = f.origChangesetId diffset = super(DiffSetManager, self).create( name=diff_file_name, revision=0, basedir=basedir, history=diffset_history, repository=repository, diffcompat=DEFAULT_DIFF_COMPAT_VERSION) if save: diffset.save() for f in files: if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data source_rev = parent_file.origInfo else: parent_content = "" if (tool.diff_uses_changeset_ids and parent_changeset_id and f.origInfo != PRE_CREATION): source_rev = parent_changeset_id else: source_rev = f.origInfo dest_file = os.path.join(basedir, f.newFile).replace("\\", "/") if f.deleted: status = FileDiff.DELETED elif f.moved: status = FileDiff.MOVED else: status = FileDiff.MODIFIED filediff = FileDiff(diffset=diffset, source_file=f.origFile, dest_file=dest_file, source_revision=smart_unicode(source_rev), dest_detail=f.newInfo, diff=f.data, parent_diff=parent_content, binary=f.binary, status=status) filediff.set_line_counts(f.insert_count, f.delete_count) if save: filediff.save() return diffset
class FileDiffMigrationTests(TestCase): """Unit tests for FileDiff migration.""" fixtures = ['test_scmtools'] def setUp(self): super(FileDiffMigrationTests, self).setUp() self.repository = self.create_repository(tool_name='Test') diffset = DiffSet.objects.create(name='test', revision=1, repository=self.repository) self.filediff = FileDiff(source_file='README', dest_file='README', diffset=diffset, diff64=b'', parent_diff64=b'') self.parent_diff = (b'diff --git a/README b/README\n' b'index 94bdd3e..3d2b777 100644\n' b'--- README\n' b'+++ README\n' b'@@ -2 +2 @@\n' b'-blah..\n' b'+blah blah\n') repository = self.create_repository(tool_name='Test') diffset = DiffSet.objects.create(name='test', revision=1, repository=repository) self.filediff = FileDiff(source_file='README', dest_file='README', diffset=diffset, diff64=b'', parent_diff64=b'') def test_migration_by_diff(self): """Testing RawFileDiffData migration accessing FileDiff.diff""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA_DIFF self.assertIsNone(self.filediff.diff_hash) self.assertIsNone(self.filediff.parent_diff_hash) # This should prompt the migration. diff = self.filediff.diff self.assertIsNone(self.filediff.parent_diff_hash) self.assertIsNotNone(self.filediff.diff_hash) self.assertEqual(diff, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(self.filediff.diff64, b'') self.assertEqual(self.filediff.diff_hash.binary, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(self.filediff.diff, diff) self.assertIsNone(self.filediff.parent_diff) self.assertIsNone(self.filediff.parent_diff_hash) def test_migration_by_parent_diff(self): """Testing RawFileDiffData migration accessing FileDiff.parent_diff""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA_DIFF self.filediff.parent_diff64 = self.parent_diff self.assertIsNone(self.filediff.parent_diff_hash) # This should prompt the migration. parent_diff = self.filediff.parent_diff self.assertIsNotNone(self.filediff.parent_diff_hash) self.assertEqual(parent_diff, self.parent_diff) self.assertEqual(self.filediff.parent_diff64, b'') self.assertEqual(self.filediff.parent_diff_hash.binary, self.parent_diff) self.assertEqual(self.filediff.parent_diff, self.parent_diff) def test_migration_by_delete_count(self): """Testing RawFileDiffData migration accessing FileDiff.delete_count""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA_DIFF self.assertIsNone(self.filediff.diff_hash) # This should prompt the migration. counts = self.filediff.get_line_counts() self.assertIsNotNone(self.filediff.diff_hash) self.assertEqual(counts['raw_delete_count'], 1) self.assertEqual(self.filediff.diff_hash.delete_count, 1) def test_migration_by_insert_count(self): """Testing RawFileDiffData migration accessing FileDiff.insert_count""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA_DIFF self.assertIsNone(self.filediff.diff_hash) # This should prompt the migration. counts = self.filediff.get_line_counts() self.assertIsNotNone(self.filediff.diff_hash) self.assertEqual(counts['raw_insert_count'], 1) self.assertEqual(self.filediff.diff_hash.insert_count, 1) def test_migration_by_set_line_counts(self): """Testing RawFileDiffData migration calling FileDiff.set_line_counts """ self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA_DIFF self.assertIsNone(self.filediff.diff_hash) # This should prompt the migration, but with our line counts. self.filediff.set_line_counts(raw_insert_count=10, raw_delete_count=20) self.assertIsNotNone(self.filediff.diff_hash) counts = self.filediff.get_line_counts() self.assertEqual(counts['raw_insert_count'], 10) self.assertEqual(counts['raw_delete_count'], 20) self.assertEqual(self.filediff.diff_hash.insert_count, 10) self.assertEqual(self.filediff.diff_hash.delete_count, 20) def test_migration_by_legacy_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.diff with associated LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.DEFAULT_GIT_FILEDIFF_DATA_DIFF)) self.filediff.legacy_diff_hash = legacy self.filediff.save() # This should prompt the migration. diff = self.filediff.diff self.assertIsNotNone(self.filediff.diff_hash) self.assertIsNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_diff_hash) self.assertEqual(LegacyFileDiffData.objects.count(), 0) self.assertEqual(diff, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(self.filediff.diff64, b'') self.assertEqual(self.filediff.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(self.filediff.diff, diff) self.assertIsNone(self.filediff.parent_diff) self.assertIsNone(self.filediff.parent_diff_hash) def test_migration_by_shared_legacy_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.diff with associated shared LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.DEFAULT_GIT_FILEDIFF_DATA_DIFF)) self.filediff.legacy_diff_hash = legacy self.filediff.save() # Create a second FileDiff using this legacy data. diffset = DiffSet.objects.create(name='test', revision=1, repository=self.repository) FileDiff.objects.create(source_file='README', dest_file='README', diffset=diffset, diff64=b'', parent_diff64=b'', legacy_diff_hash=legacy) # This should prompt the migration. diff = self.filediff.diff self.assertIsNotNone(self.filediff.diff_hash) self.assertIsNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_diff_hash) self.assertEqual(LegacyFileDiffData.objects.count(), 1) self.assertEqual(diff, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(self.filediff.diff64, b'') self.assertEqual(self.filediff.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(self.filediff.diff, diff) self.assertIsNone(self.filediff.parent_diff) self.assertIsNone(self.filediff.parent_diff_hash) def test_migration_by_legacy_parent_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.parent_diff with associated LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create(binary_hash='abc123', binary=Base64DecodedValue( self.parent_diff)) self.filediff.legacy_parent_diff_hash = legacy self.filediff.save() # This should prompt the migration. parent_diff = self.filediff.parent_diff self.assertIsNotNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_parent_diff_hash) self.assertEqual(parent_diff, self.parent_diff) self.assertEqual(self.filediff.parent_diff64, b'') self.assertEqual(self.filediff.parent_diff_hash.content, self.parent_diff) self.assertEqual(self.filediff.parent_diff, parent_diff) def test_migration_by_shared_legacy_parent_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.parent_diff with associated shared LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create(binary_hash='abc123', binary=Base64DecodedValue( self.parent_diff)) self.filediff.legacy_parent_diff_hash = legacy self.filediff.save() # Create a second FileDiff using this legacy data. diffset = DiffSet.objects.create(name='test', revision=1, repository=self.repository) FileDiff.objects.create(source_file='README', dest_file='README', diffset=diffset, diff64=b'', parent_diff64=b'', legacy_parent_diff_hash=legacy) # This should prompt the migration. parent_diff = self.filediff.parent_diff self.assertIsNotNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_parent_diff_hash) self.assertEqual(LegacyFileDiffData.objects.count(), 1) self.assertEqual(parent_diff, self.parent_diff) self.assertEqual(self.filediff.parent_diff64, b'') self.assertEqual(self.filediff.parent_diff_hash.content, self.parent_diff) self.assertEqual(self.filediff.parent_diff, parent_diff) def test_migration_with_legacy_and_race_condition(self): """Testing RawFileDiffData migration with LegacyFileDiffData and race condition in migrating """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.DEFAULT_GIT_FILEDIFF_DATA_DIFF)) parent_legacy = LegacyFileDiffData.objects.create( binary_hash='def456', binary=Base64DecodedValue(self.parent_diff)) filediff1 = self.filediff filediff1.legacy_diff_hash = legacy filediff1.legacy_parent_diff_hash = parent_legacy filediff1.save() filediff2 = FileDiff.objects.get(pk=filediff1.pk) # Make sure that we're in the expected state. self.assertEqual(filediff1.legacy_diff_hash_id, legacy.pk) self.assertEqual(filediff1.legacy_parent_diff_hash_id, parent_legacy.pk) self.assertEqual(filediff2.legacy_diff_hash_id, legacy.pk) self.assertEqual(filediff2.legacy_parent_diff_hash_id, parent_legacy.pk) # This should prompt the migration of the first instance. diff1 = self.filediff.diff parent_diff1 = filediff1.parent_diff # This should prompt the migration of the second instance. diff2 = filediff2.diff parent_diff2 = filediff2.parent_diff # At this point, we should have valid diffs, and neither call # above should have raised an exception due to a dangling hash ID. self.assertEqual(diff1, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(diff1, diff2) self.assertEqual(parent_diff1, self.parent_diff) self.assertEqual(parent_diff1, parent_diff2) self.assertEqual(LegacyFileDiffData.objects.count(), 0) self.assertEqual(RawFileDiffData.objects.count(), 2) # Check the hash references. self.assertIsNotNone(filediff1.diff_hash) self.assertIsNotNone(filediff2.diff_hash) self.assertEqual(filediff1.diff_hash, filediff2.diff_hash) self.assertIsNotNone(filediff1.parent_diff_hash) self.assertIsNotNone(filediff2.parent_diff_hash) self.assertEqual(filediff1.parent_diff_hash, filediff2.parent_diff_hash) self.assertIsNone(filediff1.legacy_diff_hash) self.assertIsNone(filediff2.legacy_diff_hash) self.assertIsNone(filediff1.legacy_parent_diff_hash) self.assertIsNone(filediff2.legacy_parent_diff_hash) # Check the diff content. self.assertEqual(filediff1.diff64, b'') self.assertEqual(filediff2.diff64, b'') self.assertEqual(filediff1.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) self.assertEqual(filediff2.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA_DIFF) # Check the parent_diff content. self.assertEqual(filediff1.parent_diff64, b'') self.assertEqual(filediff2.parent_diff64, b'') self.assertEqual(filediff1.parent_diff_hash.content, self.parent_diff) self.assertEqual(filediff2.parent_diff_hash.content, self.parent_diff)
def create(self, diff_file, parent_diff_file=None, diffset_history=None): tool = self.repository.get_scmtool() siteconfig = SiteConfiguration.objects.get_current() max_diff_size = siteconfig.get('diffviewer_max_diff_size') if max_diff_size > 0: if diff_file.size > max_diff_size: raise DiffTooBigError(_('The supplied diff file is too large'), max_diff_size=max_diff_size) if parent_diff_file and parent_diff_file.size > max_diff_size: raise DiffTooBigError( _('The supplied parent diff file is too large'), max_diff_size=max_diff_size) # Grab the base directory if there is one. if not tool.get_diffs_use_absolute_paths(): try: basedir = smart_unicode(self.cleaned_data['basedir'].strip()) except AttributeError: raise NoBaseDirError( _('The "Base Diff Path" field is required')) else: basedir = '' # Parse the diff files = list( self._process_files(diff_file, basedir, check_existance=(not parent_diff_file))) if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} # This is used only for tools like Mercurial that use atomic changeset # IDs to identify all file versions but not individual file version # IDs. parent_changeset_id = None if parent_diff_file: diff_filenames = set([f.origFile for f in files]) # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_diff_file, basedir, check_existance=True, limit_to=diff_filenames): parent_files[f.origFile] = f # Store the original changeset ID if we have it; this should # be the same for all files. if f.origChangesetId: parent_changeset_id = f.origChangesetId diffset = DiffSet(name=diff_file.name, revision=0, basedir=basedir, history=diffset_history, diffcompat=DEFAULT_DIFF_COMPAT_VERSION) diffset.repository = self.repository diffset.save() for f in files: if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data source_rev = parent_file.origInfo else: parent_content = "" if (tool.diff_uses_changeset_ids and parent_changeset_id and f.origInfo != PRE_CREATION): source_rev = parent_changeset_id else: source_rev = f.origInfo dest_file = os.path.join(basedir, f.newFile).replace("\\", "/") if f.deleted: status = FileDiff.DELETED elif f.moved: status = FileDiff.MOVED else: status = FileDiff.MODIFIED filediff = FileDiff(diffset=diffset, source_file=f.origFile, dest_file=dest_file, source_revision=smart_unicode(source_rev), dest_detail=f.newInfo, diff=f.data, parent_diff=parent_content, binary=f.binary, status=status) filediff.set_line_counts(f.insert_count, f.delete_count) filediff.save() return diffset
def create(self, diff_file, parent_diff_file=None, diffset_history=None): tool = self.repository.get_scmtool() siteconfig = SiteConfiguration.objects.get_current() max_diff_size = siteconfig.get('diffviewer_max_diff_size') if max_diff_size > 0: if diff_file.size > max_diff_size: raise DiffTooBigError( _('The supplied diff file is too large'), max_diff_size=max_diff_size) if parent_diff_file and parent_diff_file.size > max_diff_size: raise DiffTooBigError( _('The supplied parent diff file is too large'), max_diff_size=max_diff_size) # Grab the base directory if there is one. if not tool.get_diffs_use_absolute_paths(): try: basedir = smart_unicode(self.cleaned_data['basedir'].strip()) except AttributeError: raise NoBaseDirError( _('The "Base Diff Path" field is required')) else: basedir = '' # Parse the diff files = list(self._process_files( diff_file, basedir, check_existance=(not parent_diff_file))) if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} # This is used only for tools like Mercurial that use atomic changeset # IDs to identify all file versions but not individual file version # IDs. parent_changeset_id = None if parent_diff_file: # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_diff_file, basedir, check_existance=True): parent_files[f.origFile] = f # Store the original changeset ID if we have it; this should # be the same for all files. if f.origChangesetId: parent_changeset_id = f.origChangesetId diffset = DiffSet(name=diff_file.name, revision=0, basedir=basedir, history=diffset_history, diffcompat=DEFAULT_DIFF_COMPAT_VERSION) diffset.repository = self.repository diffset.save() for f in files: if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data source_rev = parent_file.origInfo else: parent_content = "" if (tool.diff_uses_changeset_ids and parent_changeset_id and f.origInfo != PRE_CREATION): source_rev = parent_changeset_id else: source_rev = f.origInfo dest_file = os.path.join(basedir, f.newFile).replace("\\", "/") if f.deleted: status = FileDiff.DELETED elif f.moved: status = FileDiff.MOVED else: status = FileDiff.MODIFIED filediff = FileDiff(diffset=diffset, source_file=f.origFile, dest_file=dest_file, source_revision=smart_unicode(source_rev), dest_detail=f.newInfo, diff=f.data, parent_diff=parent_content, binary=f.binary, status=status) filediff.save() return diffset
def create_from_data(self, repository, diff_file_name, diff_file_contents, parent_diff_file_name, parent_diff_file_contents, diffset_history, basedir, request, base_commit_id=None, save=True): """Create a DiffSet from raw diff data. The diff_file_contents and parent_diff_file_contents parameters are strings with the actual diff contents. """ from reviewboard.diffviewer.diffutils import convert_to_unicode from reviewboard.diffviewer.models import FileDiff tool = repository.get_scmtool() parser = tool.get_parser(diff_file_contents) files = list( self._process_files( parser, basedir, repository, base_commit_id, request, check_existence=(not parent_diff_file_contents))) # Parse the diff if len(files) == 0: raise EmptyDiffError(_("The diff file is empty")) # Sort the files so that header files come before implementation. files.sort(cmp=self._compare_files, key=lambda f: f.origFile) # Parse the parent diff parent_files = {} # This is used only for tools like Mercurial that use atomic changeset # IDs to identify all file versions but not individual file version # IDs. parent_commit_id = None if parent_diff_file_contents: diff_filenames = set([f.origFile for f in files]) parent_parser = tool.get_parser(parent_diff_file_contents) # If the user supplied a base diff, we need to parse it and # later apply each of the files that are in the main diff for f in self._process_files(parent_parser, basedir, repository, base_commit_id, request, check_existence=True, limit_to=diff_filenames): parent_files[f.newFile] = f # This will return a non-None value only for tools that use # commit IDs to identify file versions as opposed to file revision # IDs. parent_commit_id = parent_parser.get_orig_commit_id() diffset = self.model(name=diff_file_name, revision=0, basedir=basedir, history=diffset_history, repository=repository, diffcompat=DiffCompatVersion.DEFAULT, base_commit_id=base_commit_id) if save: diffset.save() encoding_list = repository.get_encoding_list() for f in files: parent_file = None orig_rev = None parent_content = b'' if f.origFile in parent_files: parent_file = parent_files[f.origFile] parent_content = parent_file.data orig_rev = parent_file.origInfo # If there is a parent file there is not necessarily an original # revision for the parent file in the case of a renamed file in # git. if not orig_rev: if parent_commit_id and f.origInfo != PRE_CREATION: orig_rev = parent_commit_id else: orig_rev = f.origInfo enc, orig_file = convert_to_unicode(f.origFile, encoding_list) enc, dest_file = convert_to_unicode(f.newFile, encoding_list) if f.deleted: status = FileDiff.DELETED elif f.moved: status = FileDiff.MOVED elif f.copied: status = FileDiff.COPIED else: status = FileDiff.MODIFIED filediff = FileDiff( diffset=diffset, source_file=parser.normalize_diff_filename(orig_file), dest_file=parser.normalize_diff_filename(dest_file), source_revision=smart_unicode(orig_rev), dest_detail=f.newInfo, diff=f.data, parent_diff=parent_content, binary=f.binary, status=status) if (parent_file and (parent_file.moved or parent_file.copied) and parent_file.insert_count == 0 and parent_file.delete_count == 0): filediff.extra_data = {'parent_moved': True} filediff.set_line_counts(raw_insert_count=f.insert_count, raw_delete_count=f.delete_count) if save: filediff.save() return diffset
class FileDiffMigrationTests(TestCase): """Unit tests for FileDiff migration.""" fixtures = ['test_scmtools'] def setUp(self): super(FileDiffMigrationTests, self).setUp() self.repository = self.create_repository(tool_name='Test') diffset = DiffSet.objects.create(name='test', revision=1, repository=self.repository) self.filediff = FileDiff(source_file='README', dest_file='README', diffset=diffset, diff64='', parent_diff64='') self.parent_diff = ( b'diff --git a/README b/README\n' b'index 94bdd3e..3d2b777 100644\n' b'--- README\n' b'+++ README\n' b'@@ -2 +2 @@\n' b'-blah..\n' b'+blah blah\n' ) repository = self.create_repository(tool_name='Test') diffset = DiffSet.objects.create(name='test', revision=1, repository=repository) self.filediff = FileDiff(source_file='README', dest_file='README', diffset=diffset, diff64='', parent_diff64=b'') def test_migration_by_diff(self): """Testing RawFileDiffData migration accessing FileDiff.diff""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA self.assertIsNone(self.filediff.diff_hash) self.assertIsNone(self.filediff.parent_diff_hash) # This should prompt the migration. diff = self.filediff.diff self.assertIsNone(self.filediff.parent_diff_hash) self.assertIsNotNone(self.filediff.diff_hash) self.assertEqual(diff, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(self.filediff.diff64, b'') self.assertEqual(self.filediff.diff_hash.binary, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(self.filediff.diff, diff) self.assertIsNone(self.filediff.parent_diff) self.assertIsNone(self.filediff.parent_diff_hash) def test_migration_by_parent_diff(self): """Testing RawFileDiffData migration accessing FileDiff.parent_diff""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA self.filediff.parent_diff64 = self.parent_diff self.assertIsNone(self.filediff.parent_diff_hash) # This should prompt the migration. parent_diff = self.filediff.parent_diff self.assertIsNotNone(self.filediff.parent_diff_hash) self.assertEqual(parent_diff, self.parent_diff) self.assertEqual(self.filediff.parent_diff64, b'') self.assertEqual(self.filediff.parent_diff_hash.binary, self.parent_diff) self.assertEqual(self.filediff.parent_diff, self.parent_diff) def test_migration_by_delete_count(self): """Testing RawFileDiffData migration accessing FileDiff.delete_count""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA self.assertIsNone(self.filediff.diff_hash) # This should prompt the migration. counts = self.filediff.get_line_counts() self.assertIsNotNone(self.filediff.diff_hash) self.assertEqual(counts['raw_delete_count'], 1) self.assertEqual(self.filediff.diff_hash.delete_count, 1) def test_migration_by_insert_count(self): """Testing RawFileDiffData migration accessing FileDiff.insert_count""" self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA self.assertIsNone(self.filediff.diff_hash) # This should prompt the migration. counts = self.filediff.get_line_counts() self.assertIsNotNone(self.filediff.diff_hash) self.assertEqual(counts['raw_insert_count'], 1) self.assertEqual(self.filediff.diff_hash.insert_count, 1) def test_migration_by_set_line_counts(self): """Testing RawFileDiffData migration calling FileDiff.set_line_counts """ self.filediff.diff64 = self.DEFAULT_GIT_FILEDIFF_DATA self.assertIsNone(self.filediff.diff_hash) # This should prompt the migration, but with our line counts. self.filediff.set_line_counts(raw_insert_count=10, raw_delete_count=20) self.assertIsNotNone(self.filediff.diff_hash) counts = self.filediff.get_line_counts() self.assertEqual(counts['raw_insert_count'], 10) self.assertEqual(counts['raw_delete_count'], 20) self.assertEqual(self.filediff.diff_hash.insert_count, 10) self.assertEqual(self.filediff.diff_hash.delete_count, 20) def test_migration_by_legacy_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.diff with associated LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.DEFAULT_GIT_FILEDIFF_DATA)) self.filediff.legacy_diff_hash = legacy self.filediff.save() # This should prompt the migration. diff = self.filediff.diff self.assertIsNotNone(self.filediff.diff_hash) self.assertIsNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_diff_hash) self.assertEqual(LegacyFileDiffData.objects.count(), 0) self.assertEqual(diff, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(self.filediff.diff64, '') self.assertEqual(self.filediff.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(self.filediff.diff, diff) self.assertIsNone(self.filediff.parent_diff) self.assertIsNone(self.filediff.parent_diff_hash) def test_migration_by_shared_legacy_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.diff with associated shared LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.DEFAULT_GIT_FILEDIFF_DATA)) self.filediff.legacy_diff_hash = legacy self.filediff.save() # Create a second FileDiff using this legacy data. diffset = DiffSet.objects.create(name='test', revision=1, repository=self.repository) FileDiff.objects.create(source_file='README', dest_file='README', diffset=diffset, diff64='', parent_diff64='', legacy_diff_hash=legacy) # This should prompt the migration. diff = self.filediff.diff self.assertIsNotNone(self.filediff.diff_hash) self.assertIsNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_diff_hash) self.assertEqual(LegacyFileDiffData.objects.count(), 1) self.assertEqual(diff, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(self.filediff.diff64, '') self.assertEqual(self.filediff.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(self.filediff.diff, diff) self.assertIsNone(self.filediff.parent_diff) self.assertIsNone(self.filediff.parent_diff_hash) def test_migration_by_legacy_parent_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.parent_diff with associated LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.parent_diff)) self.filediff.legacy_parent_diff_hash = legacy self.filediff.save() # This should prompt the migration. parent_diff = self.filediff.parent_diff self.assertIsNotNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_parent_diff_hash) self.assertEqual(parent_diff, self.parent_diff) self.assertEqual(self.filediff.parent_diff64, '') self.assertEqual(self.filediff.parent_diff_hash.content, self.parent_diff) self.assertEqual(self.filediff.parent_diff, parent_diff) def test_migration_by_shared_legacy_parent_diff_hash(self): """Testing RawFileDiffData migration accessing FileDiff.parent_diff with associated shared LegacyFileDiffData """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.parent_diff)) self.filediff.legacy_parent_diff_hash = legacy self.filediff.save() # Create a second FileDiff using this legacy data. diffset = DiffSet.objects.create(name='test', revision=1, repository=self.repository) FileDiff.objects.create(source_file='README', dest_file='README', diffset=diffset, diff64='', parent_diff64='', legacy_parent_diff_hash=legacy) # This should prompt the migration. parent_diff = self.filediff.parent_diff self.assertIsNotNone(self.filediff.parent_diff_hash) self.assertIsNone(self.filediff.legacy_parent_diff_hash) self.assertEqual(LegacyFileDiffData.objects.count(), 1) self.assertEqual(parent_diff, self.parent_diff) self.assertEqual(self.filediff.parent_diff64, '') self.assertEqual(self.filediff.parent_diff_hash.content, self.parent_diff) self.assertEqual(self.filediff.parent_diff, parent_diff) def test_migration_with_legacy_and_race_condition(self): """Testing RawFileDiffData migration with LegacyFileDiffData and race condition in migrating """ legacy = LegacyFileDiffData.objects.create( binary_hash='abc123', binary=Base64DecodedValue(self.DEFAULT_GIT_FILEDIFF_DATA)) parent_legacy = LegacyFileDiffData.objects.create( binary_hash='def456', binary=Base64DecodedValue(self.parent_diff)) filediff1 = self.filediff filediff1.legacy_diff_hash = legacy filediff1.legacy_parent_diff_hash = parent_legacy filediff1.save() filediff2 = FileDiff.objects.get(pk=filediff1.pk) # Make sure that we're in the expected state. self.assertEqual(filediff1.legacy_diff_hash_id, legacy.pk) self.assertEqual(filediff1.legacy_parent_diff_hash_id, parent_legacy.pk) self.assertEqual(filediff2.legacy_diff_hash_id, legacy.pk) self.assertEqual(filediff2.legacy_parent_diff_hash_id, parent_legacy.pk) # This should prompt the migration of the first instance. diff1 = self.filediff.diff parent_diff1 = filediff1.parent_diff # This should prompt the migration of the second instance. diff2 = filediff2.diff parent_diff2 = filediff2.parent_diff # At this point, we should have valid diffs, and neither call # above should have raised an exception due to a dangling hash ID. self.assertEqual(diff1, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(diff1, diff2) self.assertEqual(parent_diff1, self.parent_diff) self.assertEqual(parent_diff1, parent_diff2) self.assertEqual(LegacyFileDiffData.objects.count(), 0) self.assertEqual(RawFileDiffData.objects.count(), 2) # Check the hash references. self.assertIsNotNone(filediff1.diff_hash) self.assertIsNotNone(filediff2.diff_hash) self.assertEqual(filediff1.diff_hash, filediff2.diff_hash) self.assertIsNotNone(filediff1.parent_diff_hash) self.assertIsNotNone(filediff2.parent_diff_hash) self.assertEqual(filediff1.parent_diff_hash, filediff2.parent_diff_hash) self.assertIsNone(filediff1.legacy_diff_hash) self.assertIsNone(filediff2.legacy_diff_hash) self.assertIsNone(filediff1.legacy_parent_diff_hash) self.assertIsNone(filediff2.legacy_parent_diff_hash) # Check the diff content. self.assertEqual(filediff1.diff64, '') self.assertEqual(filediff2.diff64, '') self.assertEqual(filediff1.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA) self.assertEqual(filediff2.diff_hash.content, self.DEFAULT_GIT_FILEDIFF_DATA) # Check the parent_diff content. self.assertEqual(filediff1.parent_diff64, '') self.assertEqual(filediff2.parent_diff64, '') self.assertEqual(filediff1.parent_diff_hash.content, self.parent_diff) self.assertEqual(filediff2.parent_diff_hash.content, self.parent_diff)